POC/Provisioning: Add id mapping utility (#97997)

pkg/registry/apis/provisioning/resources/fileformat_test.go

@@ -99,11 +99,16 @@ spec:
 		info.Data, err = os.ReadFile(path.Join("../../../../..", info.Path))
 		require.NoError(t, err)
 
-		parser := NewParser(&provisioning.Repository{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "test",
+		parser := &Parser{
+			mapper: NamesFromHashedRepoPath,
+			repo: &provisioning.Repository{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+				},
 			},
-		}, &DynamicClient{}, &StaticKindsLookup{})
+			client: &DynamicClient{},
+			kinds:  &StaticKindsLookup{},
+		}
 
 		// try to validate (and lint)
 		validate := true

pkg/registry/apis/provisioning/resources/idmapper.go

@@ -0,0 +1,75 @@
+package resources
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+	"path"
+	"strings"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
+	"github.com/grafana/grafana/pkg/infra/slugify"
+)
+
+// Get a filename for a given object
+type FileNameMapper func(obj metav1.Object) string
+
+// Get the object/folder names from a path+object
+type NameMapper func(repo string, path string, obj metav1.Object) (objectName string, folderName string)
+
+// Uses the k8s uid name directly
+func FileNameFromK8sName(obj metav1.Object) string {
+	return obj.GetName() // sanitize?
+}
+
+// Calculates a human readable file name from a title field
+func FileNameFromSlug(obj metav1.Object) string {
+	meta, err := utils.MetaAccessor(obj)
+	if err == nil {
+		title := meta.FindTitle("")
+		if title != "" {
+			return slugify.Slugify(title)
+		}
+	}
+	return FileNameFromK8sName(obj)
+}
+
+// This picks the names from the saved the metadata
+func NamesFromMetadata(_ string, _ string, obj metav1.Object) (objectName string, folderName string) {
+	objectName = obj.GetName()
+	anno := obj.GetAnnotations()
+	if anno != nil {
+		folderName = anno[utils.AnnoKeyFolder]
+	}
+	return
+}
+
+// Will pick a name based on the hashed repository and path
+func NamesFromHashedRepoPath(repo string, fpath string, _ metav1.Object) (objectName string, folderName string) {
+	// remove the extension
+	idx := strings.LastIndex(fpath, ".")
+	if idx > 0 {
+		fpath = fpath[0:idx]
+	}
+
+	hasher := func(fpath string) string {
+		name := path.Base(fpath)
+		if len(name) > 12 {
+			name = name[0:12]
+		}
+		hash := sha256.New()
+		_, _ = hash.Write([]byte(repo))
+		_, _ = hash.Write([]byte{'/'})
+		_, _ = hash.Write([]byte(fpath))
+		return name + "-" + base64.URLEncoding.EncodeToString(hash.Sum(nil))[0:12]
+	}
+
+	objectName = hasher(fpath)
+
+	idx = strings.LastIndex(fpath, "/")
+	if idx > 0 {
+		folderName = hasher(fpath[0:idx])
+	}
+	return
+}

pkg/registry/apis/provisioning/resources/idmapper_test.go

@@ -0,0 +1,22 @@
+package resources
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+func TestIDMapper(t *testing.T) {
+	obj := &unstructured.Unstructured{}
+
+	dashName, folderName := NamesFromHashedRepoPath("xyz", "path/to/folder/dashboard.json", obj)
+	require.Equal(t, "dashboard-B60LqDCLR6FC", dashName)
+	require.Equal(t, "folder-aObw9Q2jRwTI", folderName)
+
+	name1, f1 := NamesFromHashedRepoPath("xyz", "path/to/folder.json", obj)
+	name2, f2 := NamesFromHashedRepoPath("xyz", "path/to/folder", obj)
+	require.Equal(t, folderName, name1)
+	require.Equal(t, folderName, name2)
+	require.Equal(t, f1, f2)
+}

pkg/registry/apis/provisioning/resources/parser.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"log/slog"
 	"path"
-	"strings"
 
 	"gopkg.in/yaml.v3"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -43,6 +42,7 @@ func (f *ParserFactory) GetParser(repo repository.Repository) (*Parser, error) {
 		repo:   config,
 		client: client,
 		kinds:  kinds,
+		mapper: NamesFromHashedRepoPath,
 	}
 	if repo.Config().Spec.Linting {
 		ctx := context.Background()
@@ -73,7 +73,8 @@ func (f *ParserFactory) GetParser(repo repository.Repository) (*Parser, error) {
 
 type Parser struct {
 	// The target repository
-	repo *provisioning.Repository
+	repo   *provisioning.Repository
+	mapper NameMapper
 
 	// client helper (for this namespace?)
 	client *DynamicClient
@@ -81,14 +82,6 @@ type Parser struct {
 	linter lint.Linter
 }
 
-func NewParser(repo *provisioning.Repository, client *DynamicClient, kinds KindsLookup) *Parser {
-	return &Parser{
-		repo:   repo,
-		client: client,
-		kinds:  kinds,
-	}
-}
-
 type ParsedResource struct {
 	// Original file Info
 	Info *repository.FileInfo
@@ -164,15 +157,9 @@ func (r *Parser) Parse(ctx context.Context, logger *slog.Logger, info *repositor
 		Timestamp: nil, // ???&info.Modified.Time,
 	})
 
-	// When name is missing use the file path as the k8s name
-	if obj.GetName() == "" {
-		name := path.Base(info.Path)
-		suffix := path.Ext(name)
-		if suffix != "" {
-			name = strings.TrimSuffix(name, suffix)
-		}
-		obj.SetName(name)
-	}
+	objName, folderName := r.mapper(cfg.Name, info.Path, obj)
+	obj.SetName(objName)
+	parsed.Meta.SetFolder(folderName)
 
 	// We can not do anything more if no kind is defined
 	if parsed.GVK == nil {

pkg/tests/apis/provisioning/provisioning_test.go

@@ -365,8 +365,6 @@ func TestIntegrationProvisioning(t *testing.T) {
 	})
 
 	t.Run("import all-panels from local-repository", func(t *testing.T) {
-		// This test will prooobably break if we change import to be an async job.
-
 		// Just make sure the folder doesn't exist in advance.
 		err := folderClient.Resource.Delete(ctx, "thisisafolderref", metav1.DeleteOptions{})
 		if err != nil && !errors.IsNotFound(err) {
@@ -431,8 +429,14 @@ func TestIntegrationProvisioning(t *testing.T) {
 			}
 		}
 
-		_, err = dashboardClient.Resource.Get(ctx, "n1jR8vnnz", metav1.GetOptions{})
-		require.NoError(t, err, "all-panels dashboard should now exist")
+		found, err := dashboardClient.Resource.List(ctx, metav1.ListOptions{})
+		require.NoError(t, err, "can list values")
+
+		names := []string{}
+		for _, v := range found.Items {
+			names = append(names, v.GetName())
+		}
+		require.Equal(t, []string{"all-panels-5Y4ReX6LwL7d"}, names, "all-panels dashboard should now exist")
 	})
 }