public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Correlations: Create paginated API (#65241)

Browse Source 
* Add pagination params and apply to sql

* Create getCorrelationsResponse that returns metadata

* Set up pagination, change correlations fetch to only get source datasource correlations

* Move correlations from root to pane, only fetch correlations for one datasource when initialized or datasource is changed

* Fix tests

* Fix remaining tests

* Use functional component to handle state

* Remove unneeded mocks, fix tests

* Change perPage to limit

* Fix Go Tests

* Fix linter

* Remove parameter

* Account for mixed datasources

* Delete unused hook

* add source UID filter to API, start backing out front end hook changes

* add source IDs to API, use when loading or changing datasource

* Fix prettier

* Mock correlations response

* Get correlations for all datasources in mixed scenario

* Add documentation for new parameters

* Attempt to fix swagger

* Fix correlations page

* add swagger and openapi docs

* Add mocks to failing test

* Change API for consistency, remove extra hooks and unused function

* Add max to limit and re-gen api docs

* Move the page to the previous page if deleting all the rows on the page

* Only fetch if remove does not have value

* Change page to a reference hook

* Fix documentation, a test and some logic thinking page could be 0
This commit is contained in:
Kristina
2023-07-05 09:37:17 -05:00
committed by GitHub
parent 340c536d0e
commit f18a02149a
31 changed files with 789 additions and 6895 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/correlations/api.go
+42 -4
View File
@@ -76,7 +76,7 @@ type CreateCorrelationParams struct {
SourceUID string `json:"sourceUID"`
}
//swagger:response createCorrelationResponse
// swagger:response createCorrelationResponse
type CreateCorrelationResponse struct {
// in: body
Body CreateCorrelationResponseBody `json:"body"`
@@ -192,7 +192,7 @@ type UpdateCorrelationParams struct {
Body UpdateCorrelationCommand `json:"body"`
}
//swagger:response updateCorrelationResponse
// swagger:response updateCorrelationResponse
type UpdateCorrelationResponse struct {
// in: body
Body UpdateCorrelationResponseBody `json:"body"`
@@ -282,7 +282,7 @@ type GetCorrelationsBySourceUIDParams struct {
DatasourceUID string `json:"sourceUID"`
}
//swagger:response getCorrelationsBySourceUIDResponse
// swagger:response getCorrelationsBySourceUIDResponse
type GetCorrelationsBySourceUIDResponse struct {
// in: body
Body []Correlation `json:"body"`
@@ -298,8 +298,25 @@ type GetCorrelationsBySourceUIDResponse struct {
// 404: notFoundError
// 500: internalServerError
func (s *CorrelationsService) getCorrelationsHandler(c *contextmodel.ReqContext) response.Response {
limit := c.QueryInt64("limit")
if limit <= 0 {
limit = 100
} else if limit > 1000 {
limit = 1000
}
page := c.QueryInt64("page")
if page <= 0 {
page = 1
}
sourceUIDs := c.QueryStrings("sourceUID")
query := GetCorrelationsQuery{
OrgId: c.OrgID,
OrgId: c.OrgID,
Limit: limit,
Page: page,
SourceUIDs: sourceUIDs,
}
correlations, err := s.getCorrelations(c.Req.Context(), query)
@@ -314,6 +331,27 @@ func (s *CorrelationsService) getCorrelationsHandler(c *contextmodel.ReqContext)
return response.JSON(http.StatusOK, correlations)
}
// swagger:parameters getCorrelations
type GetCorrelationsParams struct {
// Limit the maximum number of correlations to return per page
// in:query
// required:false
// default:100
// maximum: 1000
Limit int64 `json:"limit"`
// Page index for starting fetching correlations
// in:query
// required:false
// default:1
Page int64 `json:"page"`
// Source datasource UID filter to be applied to correlations
// in:query
// type: array
// collectionFormat: multi
// required:false
SourceUIDs []string `json:"sourceUID"`
}
//swagger:response getCorrelationsResponse
type GetCorrelationsResponse struct {
// in: body
pkg/services/correlations/correlations.go
+1 -1
View File
@@ -94,7 +94,7 @@ func (s CorrelationsService) GetCorrelationsBySourceUID(ctx context.Context, cmd
return s.getCorrelationsBySourceUID(ctx, cmd)
}
func (s CorrelationsService) GetCorrelations(ctx context.Context, cmd GetCorrelationsQuery) ([]Correlation, error) {
func (s CorrelationsService) GetCorrelations(ctx context.Context, cmd GetCorrelationsQuery) (GetCorrelationsResponseBody, error) {
return s.getCorrelations(ctx, cmd)
}
pkg/services/correlations/database.go
+34 -9
View File
@@ -225,17 +225,42 @@ func (s CorrelationsService) getCorrelationsBySourceUID(ctx context.Context, cmd
return correlations, nil
}
func (s CorrelationsService) getCorrelations(ctx context.Context, cmd GetCorrelationsQuery) ([]Correlation, error) {
correlations := make([]Correlation, 0)
err := s.SQLStore.WithDbSession(ctx, func(session *db.Session) error {
return session.Select("correlation.*").Join("", "data_source AS dss", "correlation.source_uid = dss.uid and dss.org_id = ?", cmd.OrgId).Join("", "data_source AS dst", "correlation.target_uid = dst.uid and dst.org_id = ?", cmd.OrgId).Find(&correlations)
})
if err != nil {
return []Correlation{}, err
func (s CorrelationsService) getCorrelations(ctx context.Context, cmd GetCorrelationsQuery) (GetCorrelationsResponseBody, error) {
result := GetCorrelationsResponseBody{
Correlations: make([]Correlation, 0),
Page: cmd.Page,
Limit: cmd.Limit,
}
return correlations, nil
err := s.SQLStore.WithDbSession(ctx, func(session *db.Session) error {
offset := cmd.Limit * (cmd.Page - 1)
q := session.Select("correlation.*").Join("", "data_source AS dss", "correlation.source_uid = dss.uid and dss.org_id = ?", cmd.OrgId).Join("", "data_source AS dst", "correlation.target_uid = dst.uid and dst.org_id = ?", cmd.OrgId)
if len(cmd.SourceUIDs) > 0 {
q.In("dss.uid", cmd.SourceUIDs)
}
return q.Limit(int(cmd.Limit), int(offset)).Find(&result.Correlations)
})
if err != nil {
return GetCorrelationsResponseBody{}, err
}
count, err := s.CountCorrelations(ctx)
if err != nil {
return GetCorrelationsResponseBody{}, err
}
tag, err := quota.NewTag(QuotaTargetSrv, QuotaTarget, quota.GlobalScope)
if err != nil {
return GetCorrelationsResponseBody{}, err
}
totalCount, _ := count.Get(tag)
result.TotalCount = totalCount
return result, nil
}
func (s CorrelationsService) deleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error {
pkg/services/correlations/models.go
+28 -6
View File
@@ -73,11 +73,11 @@ type CorrelationConfig struct {
Type CorrelationConfigType `json:"type" binding:"Required"`
// Target data query
// required:true
// example: { "expr": "job=app" }
// example: {"prop1":"value1","prop2":"value"}
Target map[string]interface{} `json:"target" binding:"Required"`
// Source data transformations
// required:false
// example: [{"type": "logfmt"}]
// example: [{"type":"logfmt"}]
Transformations Transformations `json:"transformations,omitempty"`
}
@@ -107,10 +107,10 @@ type Correlation struct {
// example: 50xhMlg9k
UID string `json:"uid" xorm:"pk 'uid'"`
// UID of the data source the correlation originates from
// example:d0oxYRg4z
// example: d0oxYRg4z
SourceUID string `json:"sourceUID" xorm:"pk 'source_uid'"`
// UID of the data source the correlation points to
// example:PE1C5CBDA0504A6A3
// example: PE1C5CBDA0504A6A3
TargetUID *string `json:"targetUID" xorm:"target_uid"`
// Label identifying the correlation
// example: My Label
@@ -122,6 +122,13 @@ type Correlation struct {
Config CorrelationConfig `json:"config" xorm:"jsonb config"`
}
type GetCorrelationsResponseBody struct {
Correlations []Correlation `json:"correlations"`
TotalCount int64 `json:"totalCount"`
Page int64 `json:"page"`
Limit int64 `json:"limit"`
}
// CreateCorrelationResponse is the response struct for CreateCorrelationCommand
// swagger:model
type CreateCorrelationResponseBody struct {
@@ -138,7 +145,7 @@ type CreateCorrelationCommand struct {
OrgId int64 `json:"-"`
SkipReadOnlyCheck bool `json:"-"`
// Target data source UID to which the correlation is created. required if config.type = query
// example:PE1C5CBDA0504A6A3
// example: PE1C5CBDA0504A6A3
TargetUID *string `json:"targetUID"`
// Optional label identifying the correlation
// example: My label
@@ -193,7 +200,7 @@ type CorrelationConfigUpdateDTO struct {
// Target type
Type *CorrelationConfigType `json:"type"`
// Target data query
// example: { "expr": "job=app" }
// example: {"prop1":"value1","prop2":"value"}
Target *map[string]interface{} `json:"target"`
// Source data transformations
// example: [{"type": "logfmt"},{"type":"regex","expression":"(Superman|Batman)", "variable":"name"}]
@@ -260,6 +267,21 @@ type GetCorrelationsBySourceUIDQuery struct {
// GetCorrelationsQuery is the query to retrieve all correlations
type GetCorrelationsQuery struct {
OrgId int64 `json:"-"`
// Limit the maximum number of correlations to return per page
// in:query
// required:false
// default:100
Limit int64 `json:"limit"`
// Page index for starting fetching correlations
// in:query
// required:false
// default:1
Page int64 `json:"page"`
// Source datasource UID filter to be applied to correlations
// in:query
// required:false
SourceUIDs []string `json:"sourceuid"`
}
type DeleteCorrelationsBySourceUIDCommand struct {
pkg/tests/api/correlations/common_test.go
+3 -2
View File
@@ -51,12 +51,13 @@ type User struct {
type GetParams struct {
url string
user User
page string
}
func (c TestContext) Get(params GetParams) *http.Response {
c.t.Helper()
resp, err := http.Get(c.getURL(params.url, params.user))
fmtUrl := fmt.Sprintf("%s?page=%s", params.url, params.page)
resp, err := http.Get(c.getURL(fmtUrl, params.user))
require.NoError(c.t, err)
return resp
pkg/tests/api/correlations/correlations_read_test.go
+6 -5
View File
@@ -42,17 +42,18 @@ func TestIntegrationReadCorrelation(t *testing.T) {
res := ctx.Get(GetParams{
url: "/api/datasources/correlations",
user: adminUser,
page: "0",
})
require.Equal(t, http.StatusOK, res.StatusCode)
responseBody, err := io.ReadAll(res.Body)
require.NoError(t, err)
var response []correlations.Correlation
var response correlations.GetCorrelationsResponseBody
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Len(t, response, 0)
require.Len(t, response.Correlations, 0)
require.NoError(t, res.Body.Close())
})
@@ -147,12 +148,12 @@ func TestIntegrationReadCorrelation(t *testing.T) {
responseBody, err := io.ReadAll(res.Body)
require.NoError(t, err)
var response []correlations.Correlation
var response correlations.GetCorrelationsResponseBody
err = json.Unmarshal(responseBody, &response)
require.NoError(t, err)
require.Len(t, response, 1)
require.EqualValues(t, correlation, response[0])
require.Len(t, response.Correlations, 1)
require.EqualValues(t, correlation, response.Correlations[0])
require.NoError(t, res.Body.Close())
})