feat(setting): Adding setting service client (#114428)

.github/CODEOWNERS

@@ -185,6 +185,7 @@
 /pkg/services/search/ @grafana/grafana-search-and-storage
 /pkg/services/searchusers/ @grafana/grafana-search-and-storage
 /pkg/services/secrets/ @grafana/grafana-operator-experience-squad
+/pkg/services/setting/ @grafana/grafana-backend-services-squad
 /pkg/services/shorturls/ @grafana/sharing-squad
 /pkg/services/sqlstore/ @grafana/grafana-search-and-storage
 /pkg/services/ssosettings/ @grafana/identity-squad

pkg/services/setting/service.go

@@ -0,0 +1,384 @@
+package setting
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/trace"
+	"gopkg.in/ini.v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/client-go/dynamic"
+	clientrest "k8s.io/client-go/rest"
+	"k8s.io/client-go/transport"
+
+	authlib "github.com/grafana/authlib/authn"
+	logging "github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/semconv"
+)
+
+var tracer = otel.Tracer("github.com/grafana/grafana/pkg/services/setting")
+
+const LogPrefix = "setting.service"
+
+const DefaultPageSize = int64(500)
+const DefaultQPS = float32(10)
+const DefaultBurst = 25
+
+const (
+	ApiGroup   = "setting.grafana.app"
+	apiVersion = "v0alpha1"
+	resource   = "settings"
+	kind       = "Setting"
+	listKind   = "SettingList"
+)
+
+var settingGroupVersion = schema.GroupVersionResource{
+	Group:    ApiGroup,
+	Version:  apiVersion,
+	Resource: resource,
+}
+
+var settingGroupListKind = map[schema.GroupVersionResource]string{
+	settingGroupVersion: listKind,
+}
+
+type remoteSettingServiceMetrics struct {
+	listDuration   *prometheus.HistogramVec
+	listResultSize *prometheus.HistogramVec
+}
+
+// Service retrieves configuration settings from a remote settings service.
+//
+// The service uses label selectors to filter settings. Settings are labeled with
+// "section" and "key" labels matching their spec fields.
+//
+// Example - Select all settings:
+//
+//	ctx := request.WithNamespace(context.Background(), "my-namespace")
+//	ini, err := service.ListAsIni(ctx, metav1.LabelSelector{})
+//
+// Example - Select settings from specific sections:
+//
+//	selector := metav1.LabelSelector{
+//	    MatchExpressions: []metav1.LabelSelectorRequirement{
+//	        {
+//	            Key:      "section",
+//	            Operator: metav1.LabelSelectorOpIn,
+//	            Values:   []string{"database", "server"},
+//	        },
+//	    },
+//	}
+//	ini, err := service.ListAsIni(ctx, selector)
+//
+// Example - Select settings from a single section with specific labels:
+//
+//	selector := metav1.LabelSelector{
+//	    MatchLabels: map[string]string{
+//	        "section": "database",
+//	    },
+//	}
+//	settings, err := service.List(ctx, selector)
+type Service interface {
+	prometheus.Collector
+	// ListAsIni retrieves settings filtered by a label selector from the namespace in context
+	// and returns them as an ini.File.
+	//
+	// The namespace must be present in the context, ie: via request.WithNamespace.
+	// An empty selector returns all settings in the namespace.
+	ListAsIni(ctx context.Context, selector metav1.LabelSelector) (*ini.File, error)
+
+	// List retrieves settings filtered by a label selector from the namespace in context
+	// and returns them as a slice of Setting structs.
+	//
+	// The namespace must be present in the context, ie: via request.WithNamespace.
+	// An empty selector returns all settings in the namespace.
+	List(ctx context.Context, selector metav1.LabelSelector) ([]*Setting, error)
+}
+
+type remoteSettingService struct {
+	dynamicClient dynamic.Interface
+	log           logging.Logger
+	pageSize      int64
+	metrics       remoteSettingServiceMetrics
+}
+
+var _ Service = (*remoteSettingService)(nil)
+var _ prometheus.Collector = (*remoteSettingService)(nil)
+
+// Config configures a Service.
+type Config struct {
+	// URL is the base URL for the remote settings service (required).
+	URL string
+	// TokenExchangeClient authenticates requests (required if WrapTransport is not set).
+	TokenExchangeClient *authlib.TokenExchangeClient
+	// WrapTransport wraps the HTTP transport for authentication.
+	// Takes precedence over TokenExchangeClient when both are set.
+	// At least one of WrapTransport or TokenExchangeClient is required.
+	WrapTransport transport.WrapperFunc
+	// TLSClientConfig configures TLS for the client connection.
+	TLSClientConfig clientrest.TLSClientConfig
+	// QPS limits requests per second (defaults to DefaultQPS).
+	QPS float32
+	// Burst allows request bursts above QPS (defaults to DefaultBurst).
+	Burst int
+	// PageSize sets the number of items per API page (defaults to DefaultPageSize).
+	PageSize int64
+}
+
+// Setting represents the parsed spec of a Setting resource.
+type Setting struct {
+	// Setting section
+	Section string `json:"section"`
+	// Setting key
+	Key string `json:"key"`
+	// Setting value
+	Value string `json:"value"`
+}
+
+// New creates a Service from the provided configuration.
+func New(config Config) (Service, error) {
+	log := logging.New(LogPrefix)
+	dynamicClient, err := getDynamicClient(config, log)
+	if err != nil {
+		return nil, err
+	}
+	pageSize := DefaultPageSize
+	if config.PageSize > 0 {
+		pageSize = config.PageSize
+	}
+
+	metrics := initMetrics()
+
+	return &remoteSettingService{
+		dynamicClient: dynamicClient,
+		pageSize:      pageSize,
+		log:           log,
+		metrics:       metrics,
+	}, nil
+}
+
+func (m *remoteSettingService) ListAsIni(ctx context.Context, labelSelector metav1.LabelSelector) (*ini.File, error) {
+	namespace, ok := request.NamespaceFrom(ctx)
+	ns := semconv.GrafanaNamespaceName(namespace)
+	ctx, span := tracer.Start(ctx, "remoteSettingService.ListAsIni",
+		trace.WithAttributes(ns))
+	defer span.End()
+
+	if !ok || namespace == "" {
+		return nil, tracing.Errorf(span, "missing namespace in context")
+	}
+
+	settings, err := m.List(ctx, labelSelector)
+	if err != nil {
+		return nil, err
+	}
+	iniFile, err := m.toIni(settings)
+	if err != nil {
+		return nil, tracing.Error(span, err)
+	}
+	return iniFile, nil
+}
+
+func (m *remoteSettingService) List(ctx context.Context, labelSelector metav1.LabelSelector) ([]*Setting, error) {
+	namespace, ok := request.NamespaceFrom(ctx)
+	ns := semconv.GrafanaNamespaceName(namespace)
+	ctx, span := tracer.Start(ctx, "remoteSettingService.List",
+		trace.WithAttributes(ns))
+	defer span.End()
+	if !ok || namespace == "" {
+		return nil, tracing.Errorf(span, "missing namespace in context")
+	}
+	log := m.log.FromContext(ctx).New(ns.Key, ns.Value, "function", "remoteSettingService.List", "traceId", span.SpanContext().TraceID())
+
+	startTime := time.Now()
+	var status string
+	defer func() {
+		duration := time.Since(startTime).Seconds()
+		m.metrics.listDuration.WithLabelValues(status).Observe(duration)
+	}()
+
+	selector, err := metav1.LabelSelectorAsSelector(&labelSelector)
+	if err != nil {
+		status = "error"
+		return nil, tracing.Error(span, err)
+	}
+	if selector.Empty() {
+		log.Debug("empty selector. Fetching all settings")
+	}
+
+	var allSettings []*Setting
+	var continueToken string
+	hasNext := true
+	totalPages := 0
+	// Using an upper limit to prevent infinite loops
+	for hasNext && totalPages < 1000 {
+		totalPages++
+		opts := metav1.ListOptions{
+			Limit:    m.pageSize,
+			Continue: continueToken,
+		}
+		if !selector.Empty() {
+			opts.LabelSelector = selector.String()
+		}
+
+		settingsList, lErr := m.dynamicClient.Resource(settingGroupVersion).Namespace(namespace).List(ctx, opts)
+		if lErr != nil {
+			status = "error"
+			return nil, tracing.Error(span, lErr)
+		}
+		for i := range settingsList.Items {
+			setting, pErr := parseSettingResource(&settingsList.Items[i])
+			if pErr != nil {
+				status = "error"
+				return nil, tracing.Error(span, pErr)
+			}
+			allSettings = append(allSettings, setting)
+		}
+		continueToken = settingsList.GetContinue()
+		if continueToken == "" {
+			hasNext = false
+		}
+	}
+
+	status = "success"
+	m.metrics.listResultSize.WithLabelValues(status).Observe(float64(len(allSettings)))
+
+	return allSettings, nil
+}
+
+func parseSettingResource(setting *unstructured.Unstructured) (*Setting, error) {
+	spec, found, err := unstructured.NestedMap(setting.Object, "spec")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get spec from setting: %w", err)
+	}
+	if !found {
+		return nil, fmt.Errorf("spec not found in setting %s", setting.GetName())
+	}
+
+	var result Setting
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(spec, &result); err != nil {
+		return nil, fmt.Errorf("failed to convert spec to Setting: %w", err)
+	}
+
+	return &result, nil
+}
+
+func (m *remoteSettingService) toIni(settings []*Setting) (*ini.File, error) {
+	conf := ini.Empty()
+	for _, setting := range settings {
+		if !conf.HasSection(setting.Section) {
+			_, _ = conf.NewSection(setting.Section)
+		}
+		_, err := conf.Section(setting.Section).NewKey(setting.Key, setting.Value)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return conf, nil
+}
+
+func getDynamicClient(config Config, log logging.Logger) (dynamic.Interface, error) {
+	if config.URL == "" {
+		return nil, fmt.Errorf("URL cannot be empty")
+	}
+	if config.WrapTransport == nil && config.TokenExchangeClient == nil {
+		return nil, fmt.Errorf("must set either TokenExchangeClient or WrapTransport")
+	}
+
+	wrapTransport := config.WrapTransport
+	if config.WrapTransport == nil {
+		log.Debug("using default wrapTransport with TokenExchangeClient")
+		wrapTransport = func(rt http.RoundTripper) http.RoundTripper {
+			return &authRoundTripper{
+				tokenClient: config.TokenExchangeClient,
+				transport:   rt,
+			}
+		}
+	}
+
+	qps := DefaultQPS
+	if config.QPS > 0 {
+		qps = config.QPS
+	}
+
+	burst := DefaultBurst
+	if config.Burst > 0 {
+		burst = config.Burst
+	}
+
+	return dynamic.NewForConfig(&clientrest.Config{
+		Host:            config.URL,
+		WrapTransport:   wrapTransport,
+		TLSClientConfig: config.TLSClientConfig,
+		QPS:             qps,
+		Burst:           burst,
+	})
+}
+
+// authRoundTripper wraps an HTTP transport with token-based authentication.
+type authRoundTripper struct {
+	tokenClient *authlib.TokenExchangeClient
+	transport   http.RoundTripper
+}
+
+var _ http.RoundTripper = (*authRoundTripper)(nil)
+
+func (a *authRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	token, err := a.tokenClient.Exchange(req.Context(), authlib.TokenExchangeRequest{
+		Audiences: []string{ApiGroup},
+		Namespace: "*",
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to exchange token: %w", err)
+	}
+	req = utilnet.CloneRequest(req)
+
+	req.Header.Set("X-Access-Token", fmt.Sprintf("Bearer %s", token.Token))
+	return a.transport.RoundTrip(req)
+}
+
+func initMetrics() remoteSettingServiceMetrics {
+	metrics := remoteSettingServiceMetrics{
+		listDuration: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Namespace:                   "settings",
+				Subsystem:                   "service",
+				Name:                        "list_settings_duration_seconds",
+				Help:                        "Duration of remote settings service List operations",
+				NativeHistogramBucketFactor: 1.1,
+			},
+			[]string{"status"}, // status: "success" or "error"
+		),
+		listResultSize: prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Namespace:                   "settings",
+				Subsystem:                   "service",
+				Name:                        "list_settings_result_size",
+				Help:                        "Number of settings returned by remote settings service List operations",
+				NativeHistogramBucketFactor: 1.1,
+			},
+			[]string{"status"}, // status: "success" or "error"
+		),
+	}
+	return metrics
+}
+
+func (m *remoteSettingService) Describe(descs chan<- *prometheus.Desc) {
+	m.metrics.listDuration.Describe(descs)
+	m.metrics.listResultSize.Describe(descs)
+}
+
+func (m *remoteSettingService) Collect(metrics chan<- prometheus.Metric) {
+	m.metrics.listDuration.Collect(metrics)
+	m.metrics.listResultSize.Collect(metrics)
+}

pkg/services/setting/service_test.go

@@ -0,0 +1,542 @@
+package setting
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/client-go/dynamic/fake"
+	k8testing "k8s.io/client-go/testing"
+
+	authlib "github.com/grafana/authlib/authn"
+	"github.com/grafana/grafana/pkg/infra/log"
+)
+
+func TestRemoteSettingService_ListAsIni(t *testing.T) {
+	t.Run("should filter settings by label selector", func(t *testing.T) {
+		// Create multiple settings, only some matching the selector
+		setting1 := newUnstructuredSetting("test-namespace", Setting{Section: "database", Key: "type", Value: "postgres"})
+		setting2 := newUnstructuredSetting("test-namespace", Setting{Section: "server", Key: "port", Value: "3000"})
+		setting3 := newUnstructuredSetting("test-namespace", Setting{Section: "database", Key: "host", Value: "localhost"})
+
+		client := newTestClient(500, setting1, setting2, setting3)
+
+		// Create a selector that should match only database settings
+		selector := metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"section": "database",
+			},
+		}
+
+		ctx := request.WithNamespace(context.Background(), "test-namespace")
+		result, err := client.ListAsIni(ctx, selector)
+
+		require.NoError(t, err)
+		assert.NotNil(t, result)
+		// Should only have database settings, not server settings
+		assert.True(t, result.HasSection("database"))
+		assert.Equal(t, "postgres", result.Section("database").Key("type").String())
+		assert.Equal(t, "localhost", result.Section("database").Key("host").String())
+		// Should NOT have server settings
+		assert.False(t, result.HasSection("server"))
+	})
+
+	t.Run("should return all settings with empty selector", func(t *testing.T) {
+		// Create multiple settings across different sections
+		setting1 := newUnstructuredSetting("test-namespace", Setting{Section: "server", Key: "port", Value: "3000"})
+		setting2 := newUnstructuredSetting("test-namespace", Setting{Section: "database", Key: "type", Value: "mysql"})
+
+		client := newTestClient(500, setting1, setting2)
+
+		// Empty selector should select everything
+		selector := metav1.LabelSelector{}
+
+		ctx := request.WithNamespace(context.Background(), "test-namespace")
+		result, err := client.ListAsIni(ctx, selector)
+
+		require.NoError(t, err)
+		assert.NotNil(t, result)
+		// Should have all settings from all sections
+		assert.True(t, result.HasSection("server"))
+		assert.Equal(t, "3000", result.Section("server").Key("port").String())
+		assert.True(t, result.HasSection("database"))
+		assert.Equal(t, "mysql", result.Section("database").Key("type").String())
+	})
+}
+
+func TestRemoteSettingService_List(t *testing.T) {
+	t.Run("should handle single page response", func(t *testing.T) {
+		setting := newUnstructuredSetting("test-namespace", Setting{Section: "server", Key: "port", Value: "3000"})
+
+		client := newTestClient(500, setting)
+
+		ctx := request.WithNamespace(context.Background(), "test-namespace")
+		result, err := client.List(ctx, metav1.LabelSelector{})
+
+		require.NoError(t, err)
+		assert.Len(t, result, 1)
+
+		spec := result[0]
+		assert.Equal(t, "server", spec.Section)
+		assert.Equal(t, "port", spec.Key)
+		assert.Equal(t, "3000", spec.Value)
+	})
+
+	t.Run("should handle multiple pages", func(t *testing.T) {
+		totalPages := 3
+		pageSize := 5
+
+		pages := make([][]*unstructured.Unstructured, totalPages)
+		for pageNum := 0; pageNum < totalPages; pageNum++ {
+			for idx := 0; idx < pageSize; idx++ {
+				item := newUnstructuredSetting(
+					"test-namespace",
+					Setting{
+						Section: fmt.Sprintf("section-%d", pageNum),
+						Key:     fmt.Sprintf("key-%d", idx),
+						Value:   fmt.Sprintf("val-%d-%d", pageNum, idx),
+					},
+				)
+				pages[pageNum] = append(pages[pageNum], item)
+			}
+		}
+
+		scheme := runtime.NewScheme()
+		dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(scheme, settingGroupListKind)
+		listCallCount := 0
+		dynamicClient.PrependReactor("list", "settings", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) {
+			listCallCount++
+
+			continueToken := fmt.Sprintf("continue-%d", listCallCount)
+			if listCallCount == totalPages {
+				continueToken = ""
+			}
+
+			if listCallCount <= totalPages {
+				list := &unstructured.UnstructuredList{
+					Object: map[string]interface{}{
+						"apiVersion": ApiGroup + "/" + apiVersion,
+						"kind":       listKind,
+					},
+				}
+				list.SetContinue(continueToken)
+				for _, item := range pages[listCallCount-1] {
+					list.Items = append(list.Items, *item)
+				}
+				return true, list, nil
+			}
+
+			return false, nil, nil
+		})
+
+		client := &remoteSettingService{
+			dynamicClient: dynamicClient,
+			pageSize:      int64(pageSize),
+			log:           log.NewNopLogger(),
+			metrics:       initMetrics(),
+		}
+
+		ctx := request.WithNamespace(context.Background(), "test-namespace")
+		result, err := client.List(ctx, metav1.LabelSelector{})
+
+		require.NoError(t, err)
+		assert.Len(t, result, totalPages*pageSize)
+		assert.Equal(t, totalPages, listCallCount)
+	})
+
+	t.Run("should pass label selector when provided", func(t *testing.T) {
+		scheme := runtime.NewScheme()
+		dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(scheme, settingGroupListKind)
+		dynamicClient.PrependReactor("list", "settings", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) {
+			listAction := action.(k8testing.ListActionImpl)
+			assert.Equal(t, "app=grafana", listAction.ListOptions.LabelSelector)
+			return true, &unstructured.UnstructuredList{}, nil
+		})
+
+		client := &remoteSettingService{
+			dynamicClient: dynamicClient,
+			pageSize:      500,
+			log:           log.NewNopLogger(),
+			metrics:       initMetrics(),
+		}
+
+		ctx := request.WithNamespace(context.Background(), "test-namespace")
+		_, err := client.List(ctx, metav1.LabelSelector{MatchLabels: map[string]string{"app": "grafana"}})
+
+		require.NoError(t, err)
+	})
+
+	t.Run("should stop pagination at 1000 pages", func(t *testing.T) {
+		scheme := runtime.NewScheme()
+		dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(scheme, settingGroupListKind)
+		listCallCount := 0
+		dynamicClient.PrependReactor("list", "settings", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) {
+			listCallCount++
+			// Always return a continue token to simulate infinite pagination
+			list := &unstructured.UnstructuredList{}
+			list.SetContinue("continue-forever")
+			return true, list, nil
+		})
+
+		client := &remoteSettingService{
+			dynamicClient: dynamicClient,
+			pageSize:      10,
+			log:           log.NewNopLogger(),
+			metrics:       initMetrics(),
+		}
+
+		ctx := request.WithNamespace(context.Background(), "test-namespace")
+		_, err := client.List(ctx, metav1.LabelSelector{})
+
+		require.NoError(t, err)
+		assert.Equal(t, 1000, listCallCount, "Should stop at 1000 pages to prevent infinite loops")
+	})
+
+	t.Run("should return error when parsing setting fails", func(t *testing.T) {
+		scheme := runtime.NewScheme()
+		dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(scheme, settingGroupListKind)
+		dynamicClient.PrependReactor("list", "settings", func(action k8testing.Action) (handled bool, ret runtime.Object, err error) {
+			// Return a malformed setting without spec
+			list := &unstructured.UnstructuredList{
+				Object: map[string]interface{}{
+					"apiVersion": ApiGroup + "/" + apiVersion,
+					"kind":       listKind,
+				},
+			}
+			malformedSetting := &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": ApiGroup + "/" + apiVersion,
+					"kind":       kind,
+					"metadata": map[string]interface{}{
+						"name":      "malformed",
+						"namespace": "test-namespace",
+					},
+					// Missing spec
+				},
+			}
+			list.Items = append(list.Items, *malformedSetting)
+			return true, list, nil
+		})
+
+		client := &remoteSettingService{
+			dynamicClient: dynamicClient,
+			pageSize:      500,
+			log:           log.NewNopLogger(),
+			metrics:       initMetrics(),
+		}
+
+		ctx := request.WithNamespace(context.Background(), "test-namespace")
+		result, err := client.List(ctx, metav1.LabelSelector{})
+
+		require.Error(t, err)
+		assert.Nil(t, result)
+		assert.Contains(t, err.Error(), "spec not found")
+	})
+}
+
+func TestParseSettingResource(t *testing.T) {
+	t.Run("should parse valid setting resource", func(t *testing.T) {
+		setting := newUnstructuredSetting("test-namespace", Setting{Section: "database", Key: "type", Value: "postgres"})
+
+		result, err := parseSettingResource(setting)
+
+		require.NoError(t, err)
+		assert.NotNil(t, result)
+		assert.Equal(t, "database", result.Section)
+		assert.Equal(t, "type", result.Key)
+		assert.Equal(t, "postgres", result.Value)
+	})
+
+	t.Run("should return error when spec is missing", func(t *testing.T) {
+		setting := &unstructured.Unstructured{
+			Object: map[string]interface{}{
+				"apiVersion": ApiGroup + "/" + apiVersion,
+				"kind":       kind,
+				"metadata": map[string]interface{}{
+					"name":      "test-setting",
+					"namespace": "test-namespace",
+				},
+				// No spec
+			},
+		}
+
+		result, err := parseSettingResource(setting)
+
+		require.Error(t, err)
+		assert.Nil(t, result)
+		assert.Contains(t, err.Error(), "spec not found")
+	})
+}
+
+func TestRemoteSettingService_ToIni(t *testing.T) {
+	t.Run("should convert settings to ini format", func(t *testing.T) {
+		settings := []*Setting{
+			{Section: "database", Key: "type", Value: "postgres"},
+			{Section: "database", Key: "host", Value: "localhost"},
+			{Section: "server", Key: "http_port", Value: "3000"},
+		}
+
+		client := &remoteSettingService{
+			pageSize: 500,
+			log:      log.NewNopLogger(),
+		}
+
+		result, err := client.toIni(settings)
+
+		require.NoError(t, err)
+		assert.NotNil(t, result)
+		assert.True(t, result.HasSection("database"))
+		assert.True(t, result.HasSection("server"))
+		assert.Equal(t, "postgres", result.Section("database").Key("type").String())
+		assert.Equal(t, "localhost", result.Section("database").Key("host").String())
+		assert.Equal(t, "3000", result.Section("server").Key("http_port").String())
+	})
+
+	t.Run("should handle empty settings list", func(t *testing.T) {
+		var settings []*Setting
+
+		client := &remoteSettingService{
+			pageSize: 500,
+			log:      log.NewNopLogger(),
+		}
+
+		result, err := client.toIni(settings)
+
+		require.NoError(t, err)
+		assert.NotNil(t, result)
+		sections := result.Sections()
+		assert.Len(t, sections, 1) // Only default section
+	})
+
+	t.Run("should create section if it does not exist", func(t *testing.T) {
+		settings := []*Setting{
+			{Section: "new_section", Key: "new_key", Value: "new_value"},
+		}
+
+		client := &remoteSettingService{
+			pageSize: 500,
+			log:      log.NewNopLogger(),
+		}
+
+		result, err := client.toIni(settings)
+
+		require.NoError(t, err)
+		assert.True(t, result.HasSection("new_section"))
+		assert.Equal(t, "new_value", result.Section("new_section").Key("new_key").String())
+	})
+
+	t.Run("should handle multiple keys in same section", func(t *testing.T) {
+		settings := []*Setting{
+			{Section: "auth", Key: "disable_login_form", Value: "false"},
+			{Section: "auth", Key: "disable_signout_menu", Value: "true"},
+		}
+
+		client := &remoteSettingService{
+			pageSize: 500,
+			log:      log.NewNopLogger(),
+		}
+
+		result, err := client.toIni(settings)
+
+		require.NoError(t, err)
+		assert.True(t, result.HasSection("auth"))
+		authSection := result.Section("auth")
+		assert.Equal(t, "false", authSection.Key("disable_login_form").String())
+		assert.Equal(t, "true", authSection.Key("disable_signout_menu").String())
+	})
+}
+
+func TestNew(t *testing.T) {
+	t.Run("should create client with default page size", func(t *testing.T) {
+		config := Config{
+			URL:           "https://example.com",
+			WrapTransport: func(rt http.RoundTripper) http.RoundTripper { return rt },
+		}
+
+		client, err := New(config)
+
+		require.NoError(t, err)
+		assert.NotNil(t, client)
+		remoteClient := client.(*remoteSettingService)
+		assert.Equal(t, DefaultPageSize, remoteClient.pageSize)
+	})
+
+	t.Run("should create client with custom page size", func(t *testing.T) {
+		config := Config{
+			URL:           "https://example.com",
+			WrapTransport: func(rt http.RoundTripper) http.RoundTripper { return rt },
+			PageSize:      100,
+		}
+
+		client, err := New(config)
+
+		require.NoError(t, err)
+		assert.NotNil(t, client)
+		remoteClient := client.(*remoteSettingService)
+		assert.Equal(t, int64(100), remoteClient.pageSize)
+	})
+
+	t.Run("should use default page size when zero is provided", func(t *testing.T) {
+		config := Config{
+			URL:           "https://example.com",
+			WrapTransport: func(rt http.RoundTripper) http.RoundTripper { return rt },
+			PageSize:      0,
+		}
+
+		client, err := New(config)
+
+		require.NoError(t, err)
+		assert.NotNil(t, client)
+		remoteClient := client.(*remoteSettingService)
+		assert.Equal(t, DefaultPageSize, remoteClient.pageSize)
+	})
+
+	t.Run("should return error when config is invalid", func(t *testing.T) {
+		config := Config{
+			URL: "", // Invalid: empty URL
+		}
+
+		client, err := New(config)
+
+		require.Error(t, err)
+		assert.Nil(t, client)
+		assert.Contains(t, err.Error(), "URL cannot be empty")
+	})
+}
+
+func TestGetDynamicClient(t *testing.T) {
+	logger := log.NewNopLogger()
+
+	t.Run("should return error when SettingServiceURL is empty", func(t *testing.T) {
+		config := Config{
+			URL:           "",
+			WrapTransport: func(rt http.RoundTripper) http.RoundTripper { return rt },
+		}
+
+		client, err := getDynamicClient(config, logger)
+
+		require.Error(t, err)
+		assert.Nil(t, client)
+		assert.Contains(t, err.Error(), "URL cannot be empty")
+	})
+
+	t.Run("should return error when both TokenExchangeClient and WrapTransport are nil", func(t *testing.T) {
+		config := Config{
+			URL:                 "https://example.com",
+			TokenExchangeClient: nil,
+			WrapTransport:       nil,
+		}
+
+		client, err := getDynamicClient(config, logger)
+
+		require.Error(t, err)
+		assert.Nil(t, client)
+		assert.Contains(t, err.Error(), "must set either TokenExchangeClient or WrapTransport")
+	})
+
+	t.Run("should create client with WrapTransport", func(t *testing.T) {
+		config := Config{
+			URL:           "https://example.com",
+			WrapTransport: func(rt http.RoundTripper) http.RoundTripper { return rt },
+		}
+
+		client, err := getDynamicClient(config, logger)
+
+		require.NoError(t, err)
+		assert.NotNil(t, client)
+	})
+
+	t.Run("should not fail when QPS and Burst are not provided", func(t *testing.T) {
+		config := Config{
+			URL:           "https://example.com",
+			WrapTransport: func(rt http.RoundTripper) http.RoundTripper { return rt },
+		}
+
+		client, err := getDynamicClient(config, logger)
+
+		require.NoError(t, err)
+		assert.NotNil(t, client)
+	})
+
+	t.Run("should not fail when custom QPS and Burst are provided", func(t *testing.T) {
+		config := Config{
+			URL:           "https://example.com",
+			WrapTransport: func(rt http.RoundTripper) http.RoundTripper { return rt },
+			QPS:           10.0,
+			Burst:         20,
+		}
+
+		client, err := getDynamicClient(config, logger)
+
+		require.NoError(t, err)
+		assert.NotNil(t, client)
+	})
+
+	t.Run("should use WrapTransport when both WrapTransport and TokenExchangeClient are provided", func(t *testing.T) {
+		wrapTransportCalled := false
+		tokenExchangeClient := &authlib.TokenExchangeClient{}
+
+		config := Config{
+			URL:                 "https://example.com",
+			TokenExchangeClient: tokenExchangeClient,
+			WrapTransport: func(rt http.RoundTripper) http.RoundTripper {
+				wrapTransportCalled = true
+				return rt
+			},
+		}
+
+		client, err := getDynamicClient(config, logger)
+
+		require.NoError(t, err)
+		assert.NotNil(t, client)
+		assert.True(t, wrapTransportCalled, "WrapTransport should be called and take precedence over TokenExchangeClient")
+	})
+}
+
+// Helper function to create an unstructured Setting object for tests
+func newUnstructuredSetting(namespace string, spec Setting) *unstructured.Unstructured {
+	// Generate resource name in the format {section}--{key}
+	name := fmt.Sprintf("%s--%s", spec.Section, spec.Key)
+
+	obj := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": ApiGroup + "/" + apiVersion,
+			"kind":       kind,
+			"metadata": map[string]interface{}{
+				"name":      name,
+				"namespace": namespace,
+			},
+			"spec": map[string]interface{}{
+				"section": spec.Section,
+				"key":     spec.Key,
+				"value":   spec.Value,
+			},
+		},
+	}
+	// Always set section and key labels
+	obj.SetLabels(map[string]string{
+		"section": spec.Section,
+		"key":     spec.Key,
+	})
+	return obj
+}
+
+// Helper function to create a test client with the dynamic fake client
+func newTestClient(pageSize int64, objects ...runtime.Object) *remoteSettingService {
+	scheme := runtime.NewScheme()
+	dynamicClient := fake.NewSimpleDynamicClientWithCustomListKinds(scheme, settingGroupListKind, objects...)
+
+	return &remoteSettingService{
+		dynamicClient: dynamicClient,
+		pageSize:      pageSize,
+		log:           log.NewNopLogger(),
+		metrics:       initMetrics(),
+	}
+}