Correlations: Migrate config type to root (#91855)

* WIP

* Validate new field, and add value in provisioning if not defined in correct spot

* Simplify logic, use correct value

* fix tests

* Fix linter errors

* fix swagger and tests

* 😬

* Auto-generation isnt doing this..

* Fix linter

* test if nullable is the issue…

* Change structure on the frontend fields

* Try with backtick

* try programatic quoting

* Try only quote non-ints

* quoting, no backticks

* Remove debugging

pkg/services/correlations/database.go

@@ -22,6 +22,7 @@ func (s CorrelationsService) createCorrelation(ctx context.Context, cmd CreateCo
 		Description: cmd.Description,
 		Config:      cmd.Config,
 		Provisioned: cmd.Provisioned,
+		Type:        cmd.Type,
 	}
 
 	err := s.SQLStore.WithTransactionalDbSession(ctx, func(session *db.Session) error {
@@ -131,14 +132,14 @@ func (s CorrelationsService) updateCorrelation(ctx context.Context, cmd UpdateCo
 			correlation.Description = *cmd.Description
 			session.MustCols("description")
 		}
+		if cmd.Type != nil {
+			correlation.Type = *cmd.Type
+		}
 		if cmd.Config != nil {
 			session.MustCols("config")
 			if cmd.Config.Field != nil {
 				correlation.Config.Field = *cmd.Config.Field
 			}
-			if cmd.Config.Type != nil {
-				correlation.Config.Type = *cmd.Config.Type
-			}
 			if cmd.Config.Target != nil {
 				correlation.Config.Target = *cmd.Config.Target
 			}

pkg/services/correlations/models.go

@@ -27,7 +27,7 @@ const (
 	QuotaTarget    quota.Target    = "correlations"
 )
 
-type CorrelationConfigType string
+type CorrelationType string
 
 type Transformation struct {
 	//Enum: regex,logfmt
@@ -38,11 +38,11 @@ type Transformation struct {
 }
 
 const (
-	ConfigTypeQuery CorrelationConfigType = "query"
+	TypeQuery CorrelationType = "query"
 )
 
-func (t CorrelationConfigType) Validate() error {
-	if t != ConfigTypeQuery {
+func (t CorrelationType) Validate() error {
+	if t != TypeQuery {
 		return fmt.Errorf("%s: \"%s\"", ErrInvalidConfigType, t)
 	}
 	return nil
@@ -68,8 +68,9 @@ type CorrelationConfig struct {
 	// example: message
 	Field string `json:"field" binding:"Required"`
 	// Target type
-	// required:true
-	Type CorrelationConfigType `json:"type" binding:"Required"`
+	// This is deprecated: use the type property outside of config
+	// deprecated:true
+	Type CorrelationType `json:"type"`
 	// Target data query
 	// required:true
 	// example: {"prop1":"value1","prop2":"value"}
@@ -87,12 +88,10 @@ func (c CorrelationConfig) MarshalJSON() ([]byte, error) {
 		target = map[string]any{}
 	}
 	return json.Marshal(struct {
-		Type            CorrelationConfigType `json:"type"`
-		Field           string                `json:"field"`
-		Target          map[string]any        `json:"target"`
-		Transformations Transformations       `json:"transformations,omitempty"`
+		Field           string          `json:"field"`
+		Target          map[string]any  `json:"target"`
+		Transformations Transformations `json:"transformations,omitempty"`
 	}{
-		Type:            ConfigTypeQuery,
 		Field:           c.Field,
 		Target:          target,
 		Transformations: transformations,
@@ -124,6 +123,8 @@ type Correlation struct {
 	Config CorrelationConfig `json:"config" xorm:"jsonb config"`
 	// Provisioned True if the correlation was created during provisioning
 	Provisioned bool `json:"provisioned"`
+	// The type of correlation. Currently, only valid value is "query"
+	Type CorrelationType `json:"type" binding:"Required"`
 }
 
 type GetCorrelationsResponseBody struct {
@@ -147,7 +148,7 @@ type CreateCorrelationCommand struct {
 	// UID of the data source for which correlation is created.
 	SourceUID string `json:"-"`
 	OrgId     int64  `json:"-"`
-	// Target data source UID to which the correlation is created. required if config.type = query
+	// Target data source UID to which the correlation is created. required if type = query
 	// example: PE1C5CBDA0504A6A3
 	TargetUID *string `json:"targetUID"`
 	// Optional label identifying the correlation
@@ -160,14 +161,16 @@ type CreateCorrelationCommand struct {
 	Config CorrelationConfig `json:"config" binding:"Required"`
 	// True if correlation was created with provisioning. This makes it read-only.
 	Provisioned bool `json:"provisioned"`
+	// correlation type, currently only valid value is "query"
+	Type CorrelationType `json:"type" binding:"Required"`
 }
 
 func (c CreateCorrelationCommand) Validate() error {
-	if err := c.Config.Type.Validate(); err != nil {
+	if err := c.Type.Validate(); err != nil {
 		return err
 	}
-	if c.TargetUID == nil && c.Config.Type == ConfigTypeQuery {
-		return fmt.Errorf("correlations of type \"%s\" must have a targetUID", ConfigTypeQuery)
+	if c.TargetUID == nil && c.Type == TypeQuery {
+		return fmt.Errorf("correlations of type \"%s\" must have a targetUID", TypeQuery)
 	}
 
 	if err := c.Config.Transformations.Validate(); err != nil {
@@ -202,8 +205,6 @@ type CorrelationConfigUpdateDTO struct {
 	// Field used to attach the correlation link
 	// example: message
 	Field *string `json:"field"`
-	// Target type
-	Type *CorrelationConfigType `json:"type"`
 	// Target data query
 	// example: {"prop1":"value1","prop2":"value"}
 	Target *map[string]any `json:"target"`
@@ -212,16 +213,6 @@ type CorrelationConfigUpdateDTO struct {
 	Transformations []Transformation `json:"transformations"`
 }
 
-func (c CorrelationConfigUpdateDTO) Validate() error {
-	if c.Type != nil {
-		if err := c.Type.Validate(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // UpdateCorrelationCommand is the command for updating a correlation
 // swagger:model
 type UpdateCorrelationCommand struct {
@@ -238,16 +229,12 @@ type UpdateCorrelationCommand struct {
 	Description *string `json:"description"`
 	// Correlation Configuration
 	Config *CorrelationConfigUpdateDTO `json:"config"`
+	// correlation type
+	Type *CorrelationType `json:"type"`
 }
 
 func (c UpdateCorrelationCommand) Validate() error {
-	if c.Config != nil {
-		if err := c.Config.Validate(); err != nil {
-			return err
-		}
-	}
-
-	if c.Label == nil && c.Description == nil && (c.Config == nil || (c.Config.Field == nil && c.Config.Type == nil && c.Config.Target == nil)) {
+	if c.Label == nil && c.Description == nil && c.Type == nil && (c.Config == nil || (c.Config.Field == nil && c.Config.Target == nil)) {
 		return ErrUpdateCorrelationEmptyParams
 	}
 

pkg/services/correlations/models_test.go

@@ -14,13 +14,13 @@ func TestCorrelationModels(t *testing.T) {
 			config := &CorrelationConfig{
 				Field:  "field",
 				Target: map[string]any{},
-				Type:   ConfigTypeQuery,
 			}
 			cmd := &CreateCorrelationCommand{
 				SourceUID: "some-uid",
 				OrgId:     1,
 				TargetUID: &targetUid,
 				Config:    *config,
+				Type:      TypeQuery,
 			}
 
 			require.NoError(t, cmd.Validate())
@@ -30,7 +30,7 @@ func TestCorrelationModels(t *testing.T) {
 			config := &CorrelationConfig{
 				Field:  "field",
 				Target: map[string]any{},
-				Type:   ConfigTypeQuery,
+				Type:   TypeQuery,
 			}
 			cmd := &CreateCorrelationCommand{
 				SourceUID: "some-uid",
@@ -60,7 +60,7 @@ func TestCorrelationModels(t *testing.T) {
 	t.Run("CorrelationConfigType Validate", func(t *testing.T) {
 		t.Run("Successfully validates a correct type", func(t *testing.T) {
 			type test struct {
-				input     CorrelationConfigType
+				input     CorrelationType
 				assertion require.ErrorAssertionFunc
 			}
 
@@ -79,13 +79,12 @@ func TestCorrelationModels(t *testing.T) {
 		t.Run("Applies a default empty object if target is not defined", func(t *testing.T) {
 			config := CorrelationConfig{
 				Field: "field",
-				Type:  ConfigTypeQuery,
 			}
 
 			data, err := json.Marshal(config)
 			require.NoError(t, err)
 
-			require.Equal(t, `{"type":"query","field":"field","target":{}}`, string(data))
+			require.Equal(t, `{"field":"field","target":{}}`, string(data))
 		})
 	})
 }

pkg/services/provisioning/datasources/datasources.go

@@ -192,6 +192,13 @@ func (dc *DatasourceProvisioner) applyChanges(ctx context.Context, configPath st
 }
 
 func makeCreateCorrelationCommand(correlation map[string]any, SourceUID string, OrgId int64) (correlations.CreateCorrelationCommand, error) {
+	// we look for a correlation type at the root if it is defined, if not use default
+	// we ignore the legacy config.type value - the only valid value at that version was "query"
+	var corrType = correlation["type"]
+	if corrType == nil || corrType == "" {
+		corrType = correlations.TypeQuery
+	}
+
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	createCommand := correlations.CreateCorrelationCommand{
 		SourceUID:   SourceUID,
@@ -199,6 +206,7 @@ func makeCreateCorrelationCommand(correlation map[string]any, SourceUID string,
 		Description: correlation["description"].(string),
 		OrgId:       OrgId,
 		Provisioned: true,
+		Type:        corrType.(correlations.CorrelationType),
 	}
 
 	targetUID, ok := correlation["targetUID"].(string)
@@ -222,11 +230,6 @@ func makeCreateCorrelationCommand(correlation map[string]any, SourceUID string,
 		}
 
 		createCommand.Config = config
-	} else {
-		// when provisioning correlations without config we default to type="query"
-		createCommand.Config = correlations.CorrelationConfig{
-			Type: correlations.ConfigTypeQuery,
-		}
 	}
 	if err := createCommand.Validate(); err != nil {
 		return correlations.CreateCorrelationCommand{}, err

pkg/services/sqlstore/migrations/correlations_mig.go

@@ -38,7 +38,6 @@ func addCorrelationsMigrations(mg *Migrator) {
 			// All existing records will have '0' assigned
 			{Name: "org_id", Type: DB_BigInt, IsPrimaryKey: true, Default: "0"},
 			{Name: "source_uid", Type: DB_NVarchar, Length: 40, Nullable: false, IsPrimaryKey: true},
-			// Nullable because in the future we want to have correlations to external resources
 			{Name: "target_uid", Type: DB_NVarchar, Length: 40, Nullable: true},
 			{Name: "label", Type: DB_Text, Nullable: false},
 			{Name: "description", Type: DB_Text, Nullable: false},
@@ -63,4 +62,8 @@ func addCorrelationsMigrations(mg *Migrator) {
 	mg.AddMigration("add provisioning column", NewAddColumnMigration(correlationsV2, &Column{
 		Name: "provisioned", Type: DB_Bool, Nullable: false, Default: "0",
 	}))
+
+	mg.AddMigration("add type column", NewAddColumnMigration(correlationsV2, &Column{
+		Name: "type", Type: DB_NVarchar, Length: 40, Nullable: false, Default: "'query'",
+	}))
 }

pkg/services/stats/statsimpl/stats_test.go

@@ -104,7 +104,7 @@ func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) {
 			Config: correlations.CorrelationConfig{
 				Field:  "field",
 				Target: map[string]any{},
-				Type:   correlations.ConfigTypeQuery,
+				Type:   correlations.TypeQuery,
 			},
 		}
 		correlation, err := correlationsSvc.CreateCorrelation(context.Background(), cmd)

pkg/tests/api/correlations/correlations_create_test.go

@@ -115,8 +115,8 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 			url: fmt.Sprintf("/api/datasources/uid/%s/correlations", "nonexistent-ds-uid"),
 			body: fmt.Sprintf(`{
 					"targetUID": "%s",
+					"type": "query",
 					"config": {
-						"type": "query",
 						"field": "message",
 						"target": {}
 					}
@@ -137,13 +137,13 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 		require.NoError(t, res.Body.Close())
 	})
 
-	t.Run("inexistent target data source should result in a 404 if config.type=query", func(t *testing.T) {
+	t.Run("inexistent target data source should result in a 404 if type=query", func(t *testing.T) {
 		res := ctx.Post(PostParams{
 			url: fmt.Sprintf("/api/datasources/uid/%s/correlations", writableDs),
 			body: `{
 					"targetUID": "nonexistent-uid-uid",
+					"type": "query",
 					"config": {
-						"type": "query",
 						"field": "message",
 						"target": {}
 					}
@@ -169,8 +169,8 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 			url: fmt.Sprintf("/api/datasources/uid/%s/correlations", readOnlyDS),
 			body: fmt.Sprintf(`{
 					"targetUID": "%s",
+					"type": "query",
 					"config": {
-						"type": "query",
 						"field": "message",
 						"target": {}
 					}
@@ -200,8 +200,8 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 			url: fmt.Sprintf("/api/datasources/uid/%s/correlations", writableDs),
 			body: fmt.Sprintf(`{
 					"targetUID": "%s",
+					"type": "query",
 					"config": {
-						"type": "query",
 						"field": "message",
 						"target": {}
 					}
@@ -230,7 +230,7 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 		description := "a description"
 		label := "a label"
 		fieldName := "fieldName"
-		configType := correlations.ConfigTypeQuery
+		corrType := correlations.TypeQuery
 		transformation := correlations.Transformation{Type: "logfmt"}
 		transformation2 := correlations.Transformation{Type: "regex", Expression: "testExpression", MapValue: "testVar"}
 		res := ctx.Post(PostParams{
@@ -239,8 +239,8 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 					"targetUID": "%s",
 					"description": "%s",
 					"label": "%s",
+					"type": "%s",
 					"config": {
-						"type": "%s",
 						"field": "%s",
 						"target": { "expr": "foo" },
 						"transformations": [
@@ -248,7 +248,7 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 							{"type": "regex", "expression": "testExpression", "mapValue": "testVar"}
 						]
 					}
-				}`, writableDs, description, label, configType, fieldName),
+				}`, writableDs, description, label, corrType, fieldName),
 			user: adminUser,
 		})
 		require.Equal(t, http.StatusOK, res.StatusCode)
@@ -265,7 +265,7 @@ func TestIntegrationCreateCorrelation(t *testing.T) {
 		require.Equal(t, writableDs, *response.Result.TargetUID)
 		require.Equal(t, description, response.Result.Description)
 		require.Equal(t, label, response.Result.Label)
-		require.Equal(t, configType, response.Result.Config.Type)
+		require.Equal(t, corrType, response.Result.Type)
 		require.Equal(t, fieldName, response.Result.Config.Field)
 		require.Equal(t, map[string]any{"expr": "foo"}, response.Result.Config.Target)
 		require.Equal(t, transformation, response.Result.Config.Transformations[0])

pkg/tests/api/correlations/correlations_provisioning_api_test.go

@@ -39,7 +39,7 @@ func TestIntegrationCreateOrUpdateCorrelation(t *testing.T) {
 		OrgId:     dataSource.OrgID,
 		Label:     "needs migration",
 		Config: correlations.CorrelationConfig{
-			Type:   correlations.ConfigTypeQuery,
+			Type:   correlations.TypeQuery,
 			Field:  "foo",
 			Target: map[string]any{},
 			Transformations: []correlations.Transformation{
@@ -55,7 +55,7 @@ func TestIntegrationCreateOrUpdateCorrelation(t *testing.T) {
 		OrgId:     dataSource.OrgID,
 		Label:     "existing",
 		Config: correlations.CorrelationConfig{
-			Type:   correlations.ConfigTypeQuery,
+			Type:   correlations.TypeQuery,
 			Field:  "foo",
 			Target: map[string]any{},
 			Transformations: []correlations.Transformation{

pkg/tests/api/correlations/correlations_read_test.go

@@ -77,8 +77,8 @@ func TestIntegrationReadCorrelation(t *testing.T) {
 		SourceUID: dsWithCorrelations.UID,
 		TargetUID: &dsWithCorrelations.UID,
 		OrgId:     dsWithCorrelations.OrgID,
+		Type:      correlations.TypeQuery,
 		Config: correlations.CorrelationConfig{
-			Type:   correlations.ConfigTypeQuery,
 			Field:  "foo",
 			Target: map[string]any{},
 			Transformations: []correlations.Transformation{

pkg/tests/api/correlations/correlations_update_test.go

@@ -263,9 +263,9 @@ func TestIntegrationUpdateCorrelation(t *testing.T) {
 			body: `{
 				"label": "1",
 				"description": "1",
+				"type": "query",
 				"config": {
 					"field": "field",
-					"type": "query",
 					"target": { "expr": "bar" },
 					"transformations": [ {"type": "logfmt"} ]
 				}