grafana/pkg/registry/apis/provisioning/jobs/sync/incremental.go

package sync

import (
	"context"
	"errors"
	"fmt"
	"time"

	"github.com/grafana/grafana/apps/provisioning/pkg/repository"
	"github.com/grafana/grafana/apps/provisioning/pkg/safepath"
	"github.com/grafana/grafana/pkg/infra/tracing"
	"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
	"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/trace"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
)

// Convert git changes into resource file changes
func IncrementalSync(ctx context.Context, repo repository.Versioned, previousRef, currentRef string, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer, metrics jobs.JobMetrics) error {
	syncStart := time.Now()
	if previousRef == currentRef {
		progress.SetFinalMessage(ctx, "same commit as last time")
		return nil
	}

	ctx, span := tracer.Start(ctx, "provisioning.sync.incremental")
	defer span.End()
	defer func() {
		metrics.RecordSyncDuration(jobs.SyncTypeIncremental, time.Since(syncStart))
	}()

	compareStart := time.Now()
	compareCtx, compareSpan := tracer.Start(ctx, "provisioning.sync.incremental.compare_files")
	diff, err := repo.CompareFiles(compareCtx, previousRef, currentRef)
	if err != nil {
		compareSpan.RecordError(err)
		compareSpan.End()
		return tracing.Error(span, fmt.Errorf("compare files error: %w", err))
	}
	compareSpan.End()
	metrics.RecordIncrementalSyncPhase(jobs.IncrementalSyncPhaseCompare, time.Since(compareStart))

	if len(diff) < 1 {
		progress.SetFinalMessage(ctx, "no changes detected between commits")
		return nil
	}

	progress.SetTotal(ctx, len(diff))
	progress.SetMessage(ctx, "replicating versioned changes")
	applyStart := time.Now()
	affectedFolders, err := applyIncrementalChanges(ctx, diff, repositoryResources, progress, tracer, span)
	metrics.RecordIncrementalSyncPhase(jobs.IncrementalSyncPhaseApply, time.Since(applyStart))
	if err != nil {
		return err
	}

	progress.SetMessage(ctx, "versioned changes replicated")

	if len(affectedFolders) > 0 {
		cleanupStart := time.Now()
		span.AddEvent("checking if impacted folders should be deleted", trace.WithAttributes(attribute.Int("affected_folders", len(affectedFolders))))
		err := cleanupOrphanedFolders(ctx, repo, affectedFolders, repositoryResources, tracer, progress)
		metrics.RecordIncrementalSyncPhase(jobs.IncrementalSyncPhaseCleanup, time.Since(cleanupStart))
		if err != nil {
			return tracing.Error(span, fmt.Errorf("cleanup orphaned folders: %w", err))
		}
	}

	return nil
}

func applyIncrementalChanges(ctx context.Context, diff []repository.VersionedFileChange, repositoryResources resources.RepositoryResources, progress jobs.JobProgressRecorder, tracer tracing.Tracer, span trace.Span) (affectedFolders map[string]string, err error) {
	// this will keep track of any folders that had resources deleted from it
	// with key-value as path:grafana uid.
	// after cleaning up all resources, we will look to see if the foldrs are
	// now empty, and if so, delete them.
	affectedFolders = make(map[string]string)

	for _, change := range diff {
		if ctx.Err() != nil {
			return nil, ctx.Err()
		}
		if err := progress.TooManyErrors(); err != nil {
			return nil, tracing.Error(span, err)
		}

		// Check if this resource is nested under a failed folder creation
		// This only applies to creation/update/rename operations, not deletions
		if change.Action != repository.FileActionDeleted && progress.HasDirPathFailedCreation(change.Path) {
			// Skip this resource since its parent folder failed to be created
			skipCtx, skipSpan := tracer.Start(ctx, "provisioning.sync.incremental.skip_nested_resource")
			progress.Record(skipCtx, jobs.JobResourceResult{
				Path:    change.Path,
				Action:  repository.FileActionIgnored,
				Warning: fmt.Errorf("resource was not processed because the parent folder could not be created"),
			})
			skipSpan.End()
			continue
		}

		if err := resources.IsPathSupported(change.Path); err != nil {
			ensureFolderCtx, ensureFolderSpan := tracer.Start(ctx, "provisioning.sync.incremental.ensure_folder_path_exist")
			// Maintain the safe segment for empty folders
			safeSegment := safepath.SafeSegment(change.Path)
			if !safepath.IsDir(safeSegment) {
				safeSegment = safepath.Dir(safeSegment)
			}

			if safeSegment != "" && resources.IsPathSupported(safeSegment) == nil {
				folder, err := repositoryResources.EnsureFolderPathExist(ensureFolderCtx, safeSegment)
				if err != nil {
					ensureFolderSpan.RecordError(err)
					ensureFolderSpan.End()

					progress.Record(ensureFolderCtx, jobs.JobResourceResult{
						Path:   change.Path,
						Action: repository.FileActionIgnored,
						Group:  resources.FolderKind.Group,
						Kind:   resources.FolderKind.Kind,
						Error:  err,
					})
					continue
				}

				progress.Record(ensureFolderCtx, jobs.JobResourceResult{
					Path:   safeSegment,
					Action: repository.FileActionCreated,
					Group:  resources.FolderResource.Group,
					Kind:   resources.FolderKind.Kind,
					Name:   folder,
				})
				ensureFolderSpan.End()
				continue
			}

			progress.Record(ensureFolderCtx, jobs.JobResourceResult{
				Path:   change.Path,
				Action: repository.FileActionIgnored,
			})
			ensureFolderSpan.End()
			continue
		}

		result := jobs.JobResourceResult{
			Path:   change.Path,
			Action: change.Action,
		}

		switch change.Action {
		case repository.FileActionCreated, repository.FileActionUpdated:
			writeCtx, writeSpan := tracer.Start(ctx, "provisioning.sync.incremental.write_resource_from_file")
			name, gvk, err := repositoryResources.WriteResourceFromFile(writeCtx, change.Path, change.Ref)
			if err != nil {
				writeSpan.RecordError(err)
				result.Error = fmt.Errorf("writing resource from file %s: %w", change.Path, err)
			}
			result.Name = name
			result.Kind = gvk.Kind
			result.Group = gvk.Group
			writeSpan.End()
		case repository.FileActionDeleted:
			removeCtx, removeSpan := tracer.Start(ctx, "provisioning.sync.incremental.remove_resource_from_file")
			name, folderName, gvk, err := repositoryResources.RemoveResourceFromFile(removeCtx, change.Path, change.PreviousRef)
			if err != nil {
				removeSpan.RecordError(err)
				result.Error = fmt.Errorf("removing resource from file %s: %w", change.Path, err)
			}
			result.Name = name
			result.Kind = gvk.Kind
			result.Group = gvk.Group

			if folderName != "" {
				affectedFolders[safepath.Dir(change.Path)] = folderName
			}

			removeSpan.End()
		case repository.FileActionRenamed:
			renameCtx, renameSpan := tracer.Start(ctx, "provisioning.sync.incremental.rename_resource_file")
			name, oldFolderName, gvk, err := repositoryResources.RenameResourceFile(renameCtx, change.PreviousPath, change.PreviousRef, change.Path, change.Ref)
			if err != nil {
				renameSpan.RecordError(err)
				result.Error = fmt.Errorf("renaming resource file from %s to %s: %w", change.PreviousPath, change.Path, err)
			}
			result.Name = name
			result.Kind = gvk.Kind
			result.Group = gvk.Group

			if oldFolderName != "" {
				affectedFolders[safepath.Dir(change.Path)] = oldFolderName
			}

			renameSpan.End()
		case repository.FileActionIgnored:
			// do nothing
		}
		progress.Record(ctx, result)
	}

	return affectedFolders, nil
}

// cleanupOrphanedFolders removes folders that no longer contain any resources in git after deletions have occurred.
func cleanupOrphanedFolders(
	ctx context.Context,
	repo repository.Versioned,
	affectedFolders map[string]string,
	repositoryResources resources.RepositoryResources,
	tracer tracing.Tracer,
	progress jobs.JobProgressRecorder,
) error {
	ctx, span := tracer.Start(ctx, "provisioning.sync.incremental.cleanup_orphaned_folders")
	defer span.End()

	readerRepo, ok := repo.(repository.Reader)
	if !ok {
		span.RecordError(fmt.Errorf("repository does not implement Reader"))
		return nil
	}

	for path, folderName := range affectedFolders {
		span.SetAttributes(attribute.String("folder", folderName))

		// Check if any resources under this folder failed to delete
		if progress.HasDirPathFailedDeletion(path) {
			span.AddEvent("skipping orphaned folder cleanup: a child resource in its path failed to be deleted")
			continue
		}

		// if we can no longer find the folder in git, then we can delete it from grafana
		_, err := readerRepo.Read(ctx, path, "")
		if err != nil && (errors.Is(err, repository.ErrFileNotFound) || apierrors.IsNotFound(err)) {
			span.AddEvent("folder not found in git, removing from grafana")
			if err := repositoryResources.RemoveFolder(ctx, folderName); err != nil {
				span.RecordError(err)
			} else {
				span.AddEvent("successfully deleted")
			}
			continue
		}

		span.AddEvent("folder still exists in git, continuing")
	}

	return nil
}