grafana/pkg/registry/apis/provisioning/controller/finalizers.go

package controller

import (
	"context"
	"encoding/json"
	"fmt"
	"slices"
	"sort"
	"strings"
	"sync/atomic"
	"time"

	"github.com/grafana/dskit/concurrency"
	"k8s.io/apimachinery/pkg/api/errors"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/dynamic"

	"github.com/grafana/grafana-app-sdk/logging"
	folders "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
	provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
	"github.com/grafana/grafana/apps/provisioning/pkg/repository"
	"github.com/grafana/grafana/pkg/apimachinery/utils"
	"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
	metricutils "github.com/grafana/grafana/pkg/registry/apis/provisioning/utils"
)

type finalizer struct {
	lister        resources.ResourceLister
	clientFactory resources.ClientFactory
	metrics       *finalizerMetrics
	maxWorkers    int
}

func (f *finalizer) process(ctx context.Context,
	repo repository.Repository,
	finalizers []string,
) error {
	logger := logging.FromContext(ctx)
	logger.Info("process finalizers", "finalizers", finalizers)

	orderedFinalizers := [3]string{
		repository.CleanFinalizer,
		repository.ReleaseOrphanResourcesFinalizer,
		repository.RemoveOrphanResourcesFinalizer}

	for _, finalizer := range orderedFinalizers {
		if !slices.Contains(finalizers, finalizer) {
			continue
		}
		logger.Info("running finalizer", "finalizer", finalizer)
		var err error
		var count int
		start := time.Now()
		outcome := metricutils.SuccessOutcome

		switch finalizer {
		case repository.CleanFinalizer:
			// NOTE: the controller loop will never get run unless a finalizer is set
			logger.Info("running cleanup finalizer")
			hooks, ok := repo.(repository.Hooks)
			if ok {
				if err = hooks.OnDelete(ctx); err != nil {
					err = fmt.Errorf("execute deletion hooks: %w", err)
					outcome = metricutils.ErrorOutcome
				}
			}

		case repository.ReleaseOrphanResourcesFinalizer:
			logger.Info("releasing orphan resources")
			count, err = f.processExistingItems(ctx, repo.Config(), f.releaseResources(ctx, logger))
			if err != nil {
				err = fmt.Errorf("release resources: %w", err)
				outcome = metricutils.ErrorOutcome
			}

		case repository.RemoveOrphanResourcesFinalizer:
			logger.Info("removing orphan resources")
			count, err = f.processExistingItems(ctx, repo.Config(), f.removeResources(ctx, logger))
			if err != nil {
				err = fmt.Errorf("remove resources: %w", err)
				outcome = metricutils.ErrorOutcome
			}

		default:
			logger.Error("skipping unknown finalizer", "finalizer", finalizer)
			continue
		}

		f.metrics.RecordFinalizer(finalizer, outcome, count, time.Since(start).Seconds())

		if err != nil {
			return err
		}
	}
	return nil
}

// internal iterator to walk the existing items
func (f *finalizer) processExistingItems(
	ctx context.Context,
	repo *provisioning.Repository,
	cb func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error,
) (int, error) {
	logger := logging.FromContext(ctx)
	clients, err := f.clientFactory.Clients(ctx, repo.Namespace)
	if err != nil {
		return 0, err
	}

	items, err := f.lister.List(ctx, repo.Namespace, repo.Name)
	if err != nil {
		logger.Error("error listing resources", "error", err)
		return 0, err
	}

	// Safe deletion order
	sortResourceListForDeletion(items)

	var dashboards, folderItems []*provisioning.ResourceListItem
	for _, item := range items.Items {
		if item.Group == folders.GroupVersion.Group {
			folderItems = append(folderItems, &item)
		} else {
			dashboards = append(dashboards, &item)
		}
	}

	processItem := func(jobCtx context.Context, item *provisioning.ResourceListItem) error {
		res, _, err := clients.ForResource(jobCtx, schema.GroupVersionResource{
			Group:    item.Group,
			Resource: item.Resource,
		})
		if err != nil {
			logger.Error("error getting client for resource", "resource", item.Resource, "error", err)
			return err
		}

		err = cb(res, item)
		if err != nil {
			if errors.IsNotFound(err) {
				logger.Info("resource not found, skipping", "name", item.Name, "group", item.Group, "resource", item.Resource)
				return nil
			}
			logger.Error("error processing item", "name", item.Name, "error", err)
			return fmt.Errorf("processing item: %w", err)
		}
		return nil
	}

	processGroup := func(group []*provisioning.ResourceListItem) (int, error) {
		var processed int64
		err := concurrency.ForEachJob(ctx, len(group), f.maxWorkers, func(ctx context.Context, idx int) error {
			jobCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
			defer cancel()
			item := group[idx]
			if err := processItem(jobCtx, item); err != nil {
				return err
			}
			atomic.AddInt64(&processed, 1)
			return nil
		})
		return int(processed), err
	}

	count := 0

	if len(dashboards) > 0 {
		processed, err := processGroup(dashboards)
		if err != nil {
			return processed, err
		}
		count += processed
	}

	if len(folderItems) > 0 {
		for _, item := range folderItems {
			if err := processItem(ctx, item); err != nil {
				return count, err
			}
			count++
		}
	}

	logger.Info("processed items", "items", count)
	return count, nil
}

func (f *finalizer) releaseResources(
	ctx context.Context, logger logging.Logger,
) func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
	return func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
		logger.Info("release resource",
			"name", item.Name,
			"group", item.Group,
			"resource", item.Resource,
		)

		patchAnnotations, err := getPatchedAnnotations(item)
		if err != nil {
			return fmt.Errorf("get patched annotations: %w", err)
		}

		_, err = client.Patch(
			ctx, item.Name, types.JSONPatchType, patchAnnotations, v1.PatchOptions{},
		)
		if err != nil {
			return fmt.Errorf("patch resource to release ownership: %w", err)
		}
		return nil
	}
}

func (f *finalizer) removeResources(
	ctx context.Context, logger logging.Logger,
) func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
	return func(client dynamic.ResourceInterface, item *provisioning.ResourceListItem) error {
		logger.Info("remove resource",
			"name", item.Name,
			"group", item.Group,
			"resource", item.Resource,
		)
		return client.Delete(ctx, item.Name, v1.DeleteOptions{})
	}
}

type jsonPatchOperation struct {
	Op   string `json:"op"`
	Path string `json:"path"`
}

func getPatchedAnnotations(item *provisioning.ResourceListItem) ([]byte, error) {
	annotations := []jsonPatchOperation{
		{Op: "remove", Path: "/metadata/annotations/" + escapePatchString(utils.AnnoKeyManagerKind)},
		{Op: "remove", Path: "/metadata/annotations/" + escapePatchString(utils.AnnoKeyManagerIdentity)},
	}

	if item.Path != "" {
		annotations = append(
			annotations,
			jsonPatchOperation{
				Op: "remove", Path: "/metadata/annotations/" + escapePatchString(utils.AnnoKeySourcePath),
			},
		)
	}
	if item.Hash != "" {
		annotations = append(
			annotations,
			jsonPatchOperation{
				Op: "remove", Path: "/metadata/annotations/" + escapePatchString(utils.AnnoKeySourceChecksum),
			},
		)
	}

	return json.Marshal(annotations)
}

func escapePatchString(s string) string {
	s = strings.ReplaceAll(s, "~", "~0")
	s = strings.ReplaceAll(s, "/", "~1")
	return s
}

func sortResourceListForDeletion(list *provisioning.ResourceList) {
	// FIXME: this code should be simplified once unified storage folders support recursive deletion
	// Sort by the following logic:
	// - Put folders at the end so that we empty them first.
	// - Sort folders by depth so that we remove the deepest first
	// - If the repo is created within a folder in grafana, make sure that folder is last.
	sort.Slice(list.Items, func(i, j int) bool {
		isFolderI := list.Items[i].Group == folders.GroupVersion.Group
		isFolderJ := list.Items[j].Group == folders.GroupVersion.Group

		// non-folders always go first in the order of deletion.
		if isFolderI != isFolderJ {
			return !isFolderI
		}

		// if both are not folders, keep order (doesn't matter)
		if !isFolderI && !isFolderJ {
			return false
		}

		hasFolderI := list.Items[i].Folder != ""
		hasFolderJ := list.Items[j].Folder != ""
		// if one folder is in the root (i.e. does not have a folder specified), put that last
		if hasFolderI != hasFolderJ {
			return hasFolderI
		}

		// if both are nested folder, sort by depth, with the deepest one being first
		depthI := len(strings.Split(list.Items[i].Path, "/"))
		depthJ := len(strings.Split(list.Items[j].Path, "/"))
		if depthI != depthJ {
			return depthI > depthJ
		}

		// otherwise, keep order (doesn't matter)
		return false
	})
}