Dashboards: Prevent memory leak in CUE validation by reusing context only for 100 validations (#114818)
* fix(dashboard): prevent memory leak in CUE validation by using fresh contexts Fixes #114344 The CUE validation was reusing a single cue.Context across all validations, which caused unbounded memory growth due to CUE's internal caching of intermediate computation results (disjunctions, unifications, etc.). Root Cause: - A single cue.Context was created and reused via getValidator() - Each validation added entries to the context's internal caches - These caches grew unboundedly over time - Memory could not be garbage collected because the context held references Solution: - Store the schema source string instead of a compiled cue.Value - Create a fresh cuecontext.New() for each validation - This allows the context and its caches to be garbage collected after each validation completes Performance Impact: - ~2x slower due to schema recompilation per validation - Acceptable trade-off to prevent memory leaks - Memory usage stays bounded instead of growing unboundedly * fix(dashboard): use periodic context recreation to prevent CUE memory leaks Replace fresh context creation with periodic context recreation approach. The context is reused for up to 100 validations, then recreated to allow garbage collection of cached values while maintaining good performance. This balances performance (only 19% slower than leaky approach) with memory safety (stable at ~5 MB vs ~2 GB leak). See https://github.com/grafana/grafana/issues/114344#issuecomment-3605562491 * refactor(cuevalidator): simplify to use mutex instead of atomic counter Since CUE is not thread-safe, we need the mutex for the entire validation operation anyway. Using a regular int counter protected by the mutex is simpler and cleaner than using atomic operations.
This commit is contained in:
Roberto Jiménez Sánchez
@@ -4,26 +4,65 @@ import (
|
||||
"sync"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
cuejson "cuelang.org/go/encoding/json"
|
||||
)
|
||||
|
||||
// Validator provides thread-safe CUE schema validation.
|
||||
const (
|
||||
// maxValidations limits how many validations can use the same context before it's recreated.
|
||||
// This prevents unbounded memory growth while still allowing schema reuse for performance.
|
||||
// After this many validations, the context is discarded and a new one is created.
|
||||
maxValidations = 100
|
||||
)
|
||||
|
||||
// Validator provides thread-safe CUE schema validation with periodic context recreation.
|
||||
//
|
||||
// CUE is not safe for concurrent use: https://github.com/cue-lang/cue/discussions/1205#discussioncomment-1189238
|
||||
// This validator uses a mutex to protect concurrent access to the underlying CUE validation.
|
||||
//
|
||||
// To prevent memory leaks from CUE's internal caching, we reuse a context for up to maxValidations
|
||||
// validations, then recreate it. This balances performance (schema reuse) with memory safety
|
||||
// (periodic garbage collection of cached values).
|
||||
//
|
||||
// See https://github.com/grafana/grafana/issues/114344#issuecomment-3605562491 for details
|
||||
// about the memory leak issue and this fix.
|
||||
type Validator struct {
|
||||
schema cue.Value
|
||||
mu sync.Mutex
|
||||
schemaSource string
|
||||
schemaPath cue.Path
|
||||
mu sync.Mutex
|
||||
ctx *cue.Context
|
||||
compiledSchema cue.Value
|
||||
validationCount int
|
||||
}
|
||||
|
||||
func NewValidator(schema cue.Value) *Validator {
|
||||
// NewValidatorFromSource creates a new validator from a schema source string and path.
|
||||
// This prevents memory leaks by periodically recreating the CUE context after maxValidations uses.
|
||||
func NewValidatorFromSource(schemaSource string, schemaPath cue.Path) *Validator {
|
||||
cueCtx := cuecontext.New()
|
||||
compiledSchema := cueCtx.CompileString(schemaSource).LookupPath(schemaPath)
|
||||
return &Validator{
|
||||
schema: schema,
|
||||
schemaSource: schemaSource,
|
||||
schemaPath: schemaPath,
|
||||
ctx: cueCtx,
|
||||
compiledSchema: compiledSchema,
|
||||
}
|
||||
}
|
||||
|
||||
func (v *Validator) Validate(data []byte) error {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
return cuejson.Validate(data, v.schema)
|
||||
|
||||
// Increment validation count
|
||||
v.validationCount++
|
||||
|
||||
// If we've reached the maximum number of validations, recreate the context
|
||||
if v.validationCount >= maxValidations {
|
||||
// Recreate context to allow GC to reclaim cached values
|
||||
v.ctx = cuecontext.New()
|
||||
v.compiledSchema = v.ctx.CompileString(v.schemaSource).LookupPath(v.schemaPath)
|
||||
v.validationCount = 0
|
||||
}
|
||||
|
||||
// Validate using the current compiled schema
|
||||
return cuejson.Validate(data, v.compiledSchema)
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
"cuelang.org/go/cue/errors"
|
||||
)
|
||||
|
||||
@@ -83,11 +82,13 @@ var schemaSource string
|
||||
|
||||
func getValidator() *cuevalidator.Validator {
|
||||
getSchemaOnce.Do(func() {
|
||||
cueCtx := cuecontext.New()
|
||||
compiledSchema := cueCtx.CompileString(schemaSource).LookupPath(
|
||||
// The validator uses periodic context recreation to prevent memory leaks.
|
||||
// The context is reused for up to 100 validations, then recreated to allow
|
||||
// garbage collection of cached values while maintaining good performance.
|
||||
validator = cuevalidator.NewValidatorFromSource(
|
||||
schemaSource,
|
||||
cue.ParsePath("lineage.schemas[0].schema.spec"),
|
||||
)
|
||||
validator = cuevalidator.NewValidator(compiledSchema)
|
||||
})
|
||||
|
||||
return validator
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
"cuelang.org/go/cue/errors"
|
||||
|
||||
"github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/cuevalidator"
|
||||
@@ -84,11 +83,13 @@ var schemaSource string
|
||||
|
||||
func getValidator() *cuevalidator.Validator {
|
||||
getSchemaOnce.Do(func() {
|
||||
cueCtx := cuecontext.New()
|
||||
compiledSchema := cueCtx.CompileString(schemaSource).LookupPath(
|
||||
// The validator uses periodic context recreation to prevent memory leaks.
|
||||
// The context is reused for up to 100 validations, then recreated to allow
|
||||
// garbage collection of cached values while maintaining good performance.
|
||||
validator = cuevalidator.NewValidatorFromSource(
|
||||
schemaSource,
|
||||
cue.ParsePath("lineage.schemas[0].schema.spec"),
|
||||
)
|
||||
validator = cuevalidator.NewValidator(compiledSchema)
|
||||
})
|
||||
|
||||
return validator
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
"cuelang.org/go/cue/errors"
|
||||
|
||||
"github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/cuevalidator"
|
||||
@@ -133,11 +132,13 @@ var schemaSource string
|
||||
|
||||
func getValidator() *cuevalidator.Validator {
|
||||
getSchemaOnce.Do(func() {
|
||||
cueCtx := cuecontext.New()
|
||||
compiledSchema := cueCtx.CompileString(schemaSource).LookupPath(
|
||||
// The validator uses periodic context recreation to prevent memory leaks.
|
||||
// The context is reused for up to 100 validations, then recreated to allow
|
||||
// garbage collection of cached values while maintaining good performance.
|
||||
validator = cuevalidator.NewValidatorFromSource(
|
||||
schemaSource,
|
||||
cue.ParsePath("DashboardSpec"),
|
||||
)
|
||||
validator = cuevalidator.NewValidator(compiledSchema)
|
||||
})
|
||||
|
||||
return validator
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/validation/field"
|
||||
|
||||
"cuelang.org/go/cue"
|
||||
"cuelang.org/go/cue/cuecontext"
|
||||
"cuelang.org/go/cue/errors"
|
||||
|
||||
"github.com/grafana/grafana/apps/dashboard/pkg/apis/dashboard/cuevalidator"
|
||||
@@ -133,11 +132,13 @@ var schemaSource string
|
||||
|
||||
func getValidator() *cuevalidator.Validator {
|
||||
getSchemaOnce.Do(func() {
|
||||
cueCtx := cuecontext.New()
|
||||
compiledSchema := cueCtx.CompileString(schemaSource).LookupPath(
|
||||
// The validator uses periodic context recreation to prevent memory leaks.
|
||||
// The context is reused for up to 100 validations, then recreated to allow
|
||||
// garbage collection of cached values while maintaining good performance.
|
||||
validator = cuevalidator.NewValidatorFromSource(
|
||||
schemaSource,
|
||||
cue.ParsePath("DashboardSpec"),
|
||||
)
|
||||
validator = cuevalidator.NewValidator(compiledSchema)
|
||||
})
|
||||
|
||||
return validator
|
||||
|
||||
Reference in New Issue
Block a user