public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

K8s/Annotations: Use manager/source annotations rather than repo (#101313)

Browse Source 
Co-authored-by: Stephanie Hingtgen <stephanie.hingtgen@grafana.com>
This commit is contained in:
Ryan McKinley
2025-03-05 08:54:20 +02:00
committed by GitHub
co-authored by Stephanie Hingtgen
parent e7baf9804e
commit dc2defd84f
37 changed files with 589 additions and 573 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/unified/apistore/prepare.go
+2 -13
View File
@@ -9,12 +9,13 @@ import (
"time"
"github.com/google/uuid"
authtypes "github.com/grafana/authlib/types"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/storage"
"k8s.io/klog/v2"
authtypes "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
@@ -73,12 +74,6 @@ func (s *Storage) prepareObjectForStorage(ctx context.Context, newObject runtime
obj.SetResourceVersion("")
obj.SetSelfLink("")
// Read+write will verify that repository format is accurate
repo, err := obj.GetRepositoryInfo()
if err != nil {
return nil, err
}
obj.SetRepositoryInfo(repo)
obj.SetUpdatedBy("")
obj.SetUpdatedTimestamp(nil)
obj.SetCreatedBy(info.GetUID())
@@ -136,12 +131,6 @@ func (s *Storage) prepareObjectForUpdate(ctx context.Context, updateObject runti
obj.SetDeprecatedInternalID(previousInternalID) // nolint:staticcheck
}
// Read+write will verify that origin format is accurate
repo, err := obj.GetRepositoryInfo()
if err != nil {
return nil, err
}
obj.SetRepositoryInfo(repo)
obj.SetUpdatedBy(info.GetUID())
obj.SetUpdatedTimestampMillis(time.Now().UnixMilli())
pkg/storage/unified/apistore/prepare_test.go
+23 -17
View File
@@ -6,16 +6,17 @@ import (
"time"
"github.com/bwmarrin/snowflake"
authtypes "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
"github.com/stretchr/testify/require"
"golang.org/x/exp/rand"
"k8s.io/apimachinery/pkg/api/apitesting"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apiserver/pkg/storage"
authtypes "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1"
)
var scheme = runtime.NewScheme()
@@ -87,11 +88,14 @@ func TestPrepareObjectForStorage(t *testing.T) {
meta, err := utils.MetaAccessor(obj)
require.NoError(t, err)
now := time.Now()
meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{
Name: "test-repo",
Path: "test/path",
Hash: "hash",
Timestamp: &now,
meta.SetManagerProperties(utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "test-repo",
})
meta.SetSourceProperties(utils.SourceProperties{
Path: "test/path",
Checksum: "hash",
TimestampMillis: now.UnixMilli(),
})
encodedData, err := s.prepareObjectForStorage(ctx, obj)
@@ -101,14 +105,16 @@ func TestPrepareObjectForStorage(t *testing.T) {
require.NoError(t, err)
meta, err = utils.MetaAccessor(newObject)
require.NoError(t, err)
require.Equal(t, meta.GetRepositoryHash(), "hash")
require.Equal(t, meta.GetRepositoryName(), "test-repo")
require.Equal(t, meta.GetRepositoryPath(), "test/path")
ts, err := meta.GetRepositoryTimestamp()
require.NoError(t, err)
parsed, err := time.Parse(time.RFC3339, now.UTC().Format(time.RFC3339))
require.NoError(t, err)
require.Equal(t, ts, &parsed)
m, ok := meta.GetManagerProperties()
require.True(t, ok)
s, ok := meta.GetSourceProperties()
require.True(t, ok)
require.Equal(t, m.Identity, "test-repo")
require.Equal(t, s.Checksum, "hash")
require.Equal(t, s.Path, "test/path")
require.Equal(t, s.TimestampMillis, now.UnixMilli())
})
s.opts.RequireDeprecatedInternalID = true
pkg/storage/unified/resource/document.go
+17 -6
View File
@@ -102,7 +102,10 @@ type IndexableDocument struct {
References ResourceReferences `json:"reference,omitempty"`
// When the resource is managed by an upstream repository
RepoInfo *utils.ResourceRepositoryInfo `json:"repo,omitempty"`
Manager *utils.ManagerProperties `json:"manager,omitempty"`
// When the manager knows about file paths
Source *utils.SourceProperties `json:"source,omitempty"`
}
func (m *IndexableDocument) Type() string {
@@ -173,7 +176,14 @@ func NewIndexableDocument(key *ResourceKey, rv int64, obj utils.GrafanaMetaAcces
CreatedBy: obj.GetCreatedBy(),
UpdatedBy: obj.GetUpdatedBy(),
}
doc.RepoInfo, _ = obj.GetRepositoryInfo()
m, ok := obj.GetManagerProperties()
if ok {
doc.Manager = &m
}
s, ok := obj.GetSourceProperties()
if ok {
doc.Source = &s
}
ts := obj.GetCreationTimestamp()
if !ts.Time.IsZero() {
doc.Created = ts.Time.UnixMilli()
@@ -265,10 +275,11 @@ const SEARCH_FIELD_CREATED_BY = "createdBy"
const SEARCH_FIELD_UPDATED = "updated"
const SEARCH_FIELD_UPDATED_BY = "updatedBy"
const SEARCH_FIELD_REPOSITORY_NAME = "repo.name"
const SEARCH_FIELD_REPOSITORY_PATH = "repo.path"
const SEARCH_FIELD_REPOSITORY_HASH = "repo.hash"
const SEARCH_FIELD_REPOSITORY_TIME = "repo.time"
const SEARCH_FIELD_MANAGER_KIND = "manager.kind"
const SEARCH_FIELD_MANAGER_ID = "manager.id"
const SEARCH_FIELD_SOURCE_PATH = "source.path"
const SEARCH_FIELD_SOURCE_CHECKSUM = "source.checksum"
const SEARCH_FIELD_SOURCE_TIME = "source.timestampMillis"
const SEARCH_FIELD_SCORE = "_score" // the match score
const SEARCH_FIELD_EXPLAIN = "_explain" // score explanation as JSON object
pkg/storage/unified/resource/document_test.go
+8 -5
View File
@@ -33,17 +33,20 @@ func TestStandardDocumentBuilder(t *testing.T) {
"resource": "playlists",
"name": "test1"
},
"name": "test1",
"rv": 10,
"title": "test playlist unified storage",
"title_phrase": "test playlist unified storage",
"created": 1717236672000,
"createdBy": "user:ABC",
"updatedBy": "user:XYZ",
"name": "test1",
"repo": {
"name": "something",
"manager": {
"kind": "repo",
"id": "something"
},
"source": {
"path": "path/in/system.json",
"hash": "xyz"
"checksum": "xyz"
}
}`, string(jj))
}`, string(jj))
}
pkg/storage/unified/resource/server.go
+3 -6
View File
@@ -456,12 +456,9 @@ func (s *server) newEvent(ctx context.Context, user claims.AuthInfo, key *Resour
}
}
repo, err := obj.GetRepositoryInfo()
if err != nil {
return nil, NewBadRequestError("invalid repository info")
}
if repo != nil {
err = s.writeHooks.CanWriteValueFromRepository(ctx, user, repo.Name)
m, ok := obj.GetManagerProperties()
if ok && m.Kind == utils.ManagerKindRepo {
err = s.writeHooks.CanWriteValueFromRepository(ctx, user, m.Identity)
if err != nil {
return nil, AsErrorResult(err)
}
pkg/storage/unified/search/bleve.go
+18 -12
View File
@@ -18,11 +18,12 @@ import (
"github.com/blevesearch/bleve/v2/search/query"
bleveSearch "github.com/blevesearch/bleve/v2/search/searcher"
index "github.com/blevesearch/bleve_index_api"
"github.com/grafana/grafana/pkg/services/dashboards/dashboardaccess"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"go.opentelemetry.io/otel/trace"
"k8s.io/apimachinery/pkg/selection"
"github.com/grafana/grafana/pkg/services/dashboards/dashboardaccess"
"github.com/grafana/grafana/pkg/services/featuremgmt"
authlib "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/log"
@@ -304,19 +305,20 @@ func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.Li
found, err := b.index.SearchInContext(ctx, &bleve.SearchRequest{
Query: &query.TermQuery{
Term: req.Name,
FieldVal: resource.SEARCH_FIELD_REPOSITORY_NAME,
FieldVal: resource.SEARCH_FIELD_MANAGER_ID,
},
Fields: []string{
resource.SEARCH_FIELD_TITLE,
resource.SEARCH_FIELD_FOLDER,
resource.SEARCH_FIELD_REPOSITORY_NAME,
resource.SEARCH_FIELD_REPOSITORY_PATH,
resource.SEARCH_FIELD_REPOSITORY_HASH,
resource.SEARCH_FIELD_REPOSITORY_TIME,
resource.SEARCH_FIELD_MANAGER_KIND,
resource.SEARCH_FIELD_MANAGER_ID,
resource.SEARCH_FIELD_SOURCE_PATH,
resource.SEARCH_FIELD_SOURCE_CHECKSUM,
resource.SEARCH_FIELD_SOURCE_TIME,
},
Sort: search.SortOrder{
&search.SortField{
Field: resource.SEARCH_FIELD_REPOSITORY_PATH,
Field: resource.SEARCH_FIELD_SOURCE_PATH,
Type: search.SortFieldAsString,
Desc: false,
},
@@ -347,6 +349,10 @@ func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.Li
if ok {
return intV
}
floatV, ok := v.(float64)
if ok {
return int64(floatV)
}
str, ok := v.(string)
if ok {
t, _ := time.Parse(time.RFC3339, str)
@@ -359,9 +365,9 @@ func (b *bleveIndex) ListRepositoryObjects(ctx context.Context, req *resource.Li
for _, hit := range found.Hits {
item := &resource.ListRepositoryObjectsResponse_Item{
Object: &resource.ResourceKey{},
Hash: asString(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_HASH]),
Path: asString(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_PATH]),
Time: asTime(hit.Fields[resource.SEARCH_FIELD_REPOSITORY_TIME]),
Hash: asString(hit.Fields[resource.SEARCH_FIELD_SOURCE_CHECKSUM]),
Path: asString(hit.Fields[resource.SEARCH_FIELD_SOURCE_PATH]),
Time: asTime(hit.Fields[resource.SEARCH_FIELD_SOURCE_TIME]),
Title: asString(hit.Fields[resource.SEARCH_FIELD_TITLE]),
Folder: asString(hit.Fields[resource.SEARCH_FIELD_FOLDER]),
}
@@ -379,7 +385,7 @@ func (b *bleveIndex) CountRepositoryObjects(ctx context.Context) ([]*resource.Co
Query: bleve.NewMatchAllQuery(),
Size: 0,
Facets: bleve.FacetsRequest{
"count": bleve.NewFacetRequest(resource.SEARCH_FIELD_REPOSITORY_NAME, 1000), // typically less then 5
"count": bleve.NewFacetRequest(resource.SEARCH_FIELD_MANAGER_ID, 1000), // typically less then 5
},
})
if err != nil {
pkg/storage/unified/search/bleve_mappings.go
+20 -8
View File
@@ -69,9 +69,9 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
mapper.AddFieldMappingsAt(resource.SEARCH_FIELD_FOLDER, folderMapping)
// Repositories
repo := bleve.NewDocumentStaticMapping()
repo.AddFieldMappingsAt("name", &mapping.FieldMapping{
Name: "name",
manager := bleve.NewDocumentStaticMapping()
manager.AddFieldMappingsAt("kind", &mapping.FieldMapping{
Name: "kind",
Type: "text",
Analyzer: keyword.Name,
Store: true,
@@ -79,7 +79,18 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
})
repo.AddFieldMappingsAt("path", &mapping.FieldMapping{
manager.AddFieldMappingsAt("id", &mapping.FieldMapping{
Name: "id",
Type: "text",
Analyzer: keyword.Name,
Store: true,
Index: true,
IncludeTermVectors: false,
IncludeInAll: true,
})
source := bleve.NewDocumentStaticMapping()
source.AddFieldMappingsAt("path", &mapping.FieldMapping{
Name: "path",
Type: "text",
Analyzer: keyword.Name,
@@ -88,8 +99,8 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
})
repo.AddFieldMappingsAt("hash", &mapping.FieldMapping{
Name: "hash",
source.AddFieldMappingsAt("checksum", &mapping.FieldMapping{
Name: "checksum",
Type: "text",
Analyzer: keyword.Name,
Store: true,
@@ -97,9 +108,10 @@ func getBleveDocMappings(_ resource.SearchableDocumentFields) *mapping.DocumentM
IncludeTermVectors: false,
IncludeInAll: true,
})
repo.AddFieldMappingsAt("time", mapping.NewDateTimeFieldMapping())
source.AddFieldMappingsAt("timestampMillis", mapping.NewNumericFieldMapping())
mapper.AddSubDocumentMapping("repo", repo)
mapper.AddSubDocumentMapping("manager", manager)
mapper.AddSubDocumentMapping("source", source)
labelMapper := bleve.NewDocumentMapping()
mapper.AddSubDocumentMapping(resource.SEARCH_FIELD_LABELS, labelMapper)
pkg/storage/unified/search/bleve_mappings_test.go
+9 -6
View File
@@ -25,11 +25,14 @@ func TestDocumentMapping(t *testing.T) {
"x": "y",
},
RV: 1234,
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "nnn",
Path: "ppp",
Hash: "hhh",
Timestamp: asTimePointer(1234),
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "rrr",
},
Source: &utils.SourceProperties{
Path: "ppp",
Checksum: "ooo",
TimestampMillis: 1234,
},
}
@@ -43,5 +46,5 @@ func TestDocumentMapping(t *testing.T) {
fmt.Printf("DOC: fields %d\n", len(doc.Fields))
fmt.Printf("DOC: size %d\n", doc.Size())
require.Equal(t, 13, len(doc.Fields))
require.Equal(t, 14, len(doc.Fields))
}
pkg/storage/unified/search/bleve_test.go
+33 -30
View File
@@ -7,20 +7,18 @@ import (
"os"
"path/filepath"
"testing"
"time"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
authlib "github.com/grafana/authlib/types"
"github.com/grafana/grafana/pkg/apimachinery/identity"
authzextv1 "github.com/grafana/grafana/pkg/services/authz/proto/v1"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/apimachinery/utils"
"github.com/grafana/grafana/pkg/infra/tracing"
authzextv1 "github.com/grafana/grafana/pkg/services/authz/proto/v1"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
@@ -90,11 +88,14 @@ func TestBleveBackend(t *testing.T) {
utils.LabelKeyDeprecatedInternalID: "10", // nolint:staticcheck
},
Tags: []string{"aa", "bb"},
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/aaa.json",
Hash: "xyz",
Timestamp: asTimePointer(1609462800000), // 2021
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo-1",
},
Source: &utils.SourceProperties{
Path: "path/to/aaa.json",
Checksum: "xyz",
TimestampMillis: 1609462800000, // 2021
},
})
_ = index.Write(&resource.IndexableDocument{
@@ -119,11 +120,14 @@ func TestBleveBackend(t *testing.T) {
"region": "east",
utils.LabelKeyDeprecatedInternalID: "11", // nolint:staticcheck
},
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/bbb.json",
Hash: "hijk",
Timestamp: asTimePointer(1640998800000), // 2022
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo-1",
},
Source: &utils.SourceProperties{
Path: "path/to/bbb.json",
Checksum: "hijk",
TimestampMillis: 1640998800000, // 2022
},
})
_ = index.Write(&resource.IndexableDocument{
@@ -138,8 +142,11 @@ func TestBleveBackend(t *testing.T) {
Title: "ccc (dash)",
TitlePhrase: "ccc (dash)",
Folder: "zzz",
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo2",
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo2",
},
Source: &utils.SourceProperties{
Path: "path/in/repo2.yaml",
},
Fields: map[string]any{},
@@ -263,6 +270,7 @@ func TestBleveBackend(t *testing.T) {
jj, err := json.MarshalIndent(found, "", " ")
require.NoError(t, err)
fmt.Printf("%s\n", string(jj))
// NOTE "hash" -> "checksum" requires changing the protobuf
require.JSONEq(t, `{
"items": [
{
@@ -334,11 +342,14 @@ func TestBleveBackend(t *testing.T) {
},
Title: "zzz (folder)",
TitlePhrase: "zzz (folder)",
RepoInfo: &utils.ResourceRepositoryInfo{
Name: "repo-1",
Path: "path/to/folder.json",
Hash: "xxxx",
Timestamp: asTimePointer(300),
Manager: &utils.ManagerProperties{
Kind: utils.ManagerKindRepo,
Identity: "repo-1",
},
Source: &utils.SourceProperties{
Path: "path/to/folder.json",
Checksum: "xxxx",
TimestampMillis: 300,
},
})
_ = index.Write(&resource.IndexableDocument{
@@ -559,14 +570,6 @@ func TestGetSortFields(t *testing.T) {
})
}
func asTimePointer(milli int64) *time.Time {
if milli > 0 {
t := time.UnixMilli(milli)
return &t
}
return nil
}
var _ authlib.AccessClient = (*StubAccessClient)(nil)
func NewStubAccessClient(permissions map[string]bool) *StubAccessClient {
pkg/storage/unified/search/document_test.go
+15 -3
View File
@@ -81,14 +81,26 @@ func TestDashboardDocumentBuilder(t *testing.T) {
// Standard
builder = resource.StandardDocumentBuilder()
doSnapshotTests(t, builder, "folder", key, []string{
doSnapshotTests(t, builder, "folder", &resource.ResourceKey{
Namespace: "default",
Group: "folder.grafana.app",
Resource: "folders",
}, []string{
"aaa",
"bbb",
})
doSnapshotTests(t, builder, "playlist", key, []string{
doSnapshotTests(t, builder, "playlist", &resource.ResourceKey{
Namespace: "default",
Group: "playlist.grafana.app",
Resource: "playlists",
}, []string{
"aaa",
})
doSnapshotTests(t, builder, "report", key, []string{
doSnapshotTests(t, builder, "report", &resource.ResourceKey{
Namespace: "default",
Group: "reporting.grafana.app",
Resource: "reports",
}, []string{
"aaa",
})
}
pkg/storage/unified/search/testdata/doc/folder-aaa-out.json
Vendored
+5 -4
View File
@@ -1,8 +1,8 @@
{
"key": {
"namespace": "default",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"group": "folder.grafana.app",
"resource": "folders",
"name": "aaa"
},
"name": "aaa",
@@ -11,7 +11,8 @@
"title_phrase": "test-aaa",
"created": 1730490142000,
"createdBy": "user:1",
"repo": {
"name": "SQL"
"manager": {
"kind": "repo",
"id": "MyGIT"
}
}
pkg/storage/unified/search/testdata/doc/folder-aaa.json
Vendored
+1 -1
View File
@@ -8,7 +8,7 @@
"creationTimestamp": "2024-11-01T19:42:22Z",
"annotations": {
"grafana.app/createdBy": "user:1",
"grafana.app/originName": "SQL"
"grafana.app/repoName": "MyGIT"
}
},
"spec": {
pkg/storage/unified/search/testdata/doc/folder-bbb-out.json
Vendored
+5 -4
View File
@@ -1,8 +1,8 @@
{
"key": {
"namespace": "default",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"group": "folder.grafana.app",
"resource": "folders",
"name": "bbb"
},
"name": "bbb",
@@ -11,7 +11,8 @@
"title_phrase": "test-bbb",
"created": 1730490142000,
"createdBy": "user:1",
"repo": {
"name": "SQL"
"manager": {
"kind": "repo",
"id": "MyGIT"
}
}
pkg/storage/unified/search/testdata/doc/folder-bbb.json
Vendored
+1 -1
View File
@@ -8,7 +8,7 @@
"creationTimestamp": "2024-11-01T19:42:22Z",
"annotations": {
"grafana.app/createdBy": "user:1",
"grafana.app/originName": "SQL"
"grafana.app/repoName": "MyGIT"
}
},
"spec": {
pkg/storage/unified/search/testdata/doc/playlist-aaa-out.json
Vendored
+3 -8
View File
@@ -1,8 +1,8 @@
{
"key": {
"namespace": "default",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"group": "playlist.grafana.app",
"resource": "playlists",
"name": "aaa"
},
"name": "aaa",
@@ -10,10 +10,5 @@
"title": "Test AAA",
"title_phrase": "test aaa",
"created": 1731336353000,
"createdBy": "user:t000000001",
"repo": {
"name": "UI",
"path": "/playlists/new",
"hash": "Grafana v11.4.0-pre (c0de407fee)"
}
"createdBy": "user:t000000001"
}
pkg/storage/unified/search/testdata/doc/report-aaa-out.json
Vendored
+2 -2
View File
@@ -1,8 +1,8 @@
{
"key": {
"namespace": "default",
"group": "dashboard.grafana.app",
"resource": "dashboards",
"group": "reporting.grafana.app",
"resource": "reports",
"name": "aaa"
},
"name": "aaa",