public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Anonymous Access: Pagination for devices (#80028)

Browse Source 
* first commit

* add: pagination to anondevices

* fmt

* swagger and tests

* swagger

* testing out test

* fixing tests

* made it possible to query for from and to time

* refactor: change to query for ip adress instead

* fix: tests
This commit is contained in:
Eric Leijonmarck
2024-01-15 12:13:38 +00:00
committed by GitHub
parent e2b706fdd3
commit 3979ea0c47
14 changed files with 752 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/anonymous/anonimpl/anonstore/database.go
+88
View File
@@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/search/model"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
)
@@ -32,6 +33,30 @@ type Device struct {
UpdatedAt time.Time `json:"updatedAt" xorm:"updated_at" db:"updated_at"`
}
type DeviceSearchHitDTO struct {
DeviceID string `json:"deviceId" xorm:"device_id" db:"device_id"`
ClientIP string `json:"clientIp" xorm:"client_ip" db:"client_ip"`
UserAgent string `json:"userAgent" xorm:"user_agent" db:"user_agent"`
CreatedAt time.Time `json:"createdAt" xorm:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updatedAt" xorm:"updated_at" db:"updated_at"`
LastSeenAt time.Time `json:"lastSeenAt"`
}
type SearchDeviceQueryResult struct {
TotalCount int64 `json:"totalCount"`
Devices []*DeviceSearchHitDTO `json:"devices"`
Page int `json:"page"`
PerPage int `json:"perPage"`
}
type SearchDeviceQuery struct {
Query string
Page int
Limit int
From time.Time
To time.Time
SortOpts []model.SortOption
}
func (a *Device) CacheKey() string {
return strings.Join([]string{cacheKeyPrefix, a.DeviceID}, ":")
}
@@ -47,6 +72,8 @@ type AnonStore interface {
DeleteDevice(ctx context.Context, deviceID string) error
// DeleteDevicesOlderThan deletes all devices that have no been updated since the given time.
DeleteDevicesOlderThan(ctx context.Context, olderThan time.Time) error
// SearchDevices searches for devices within the 30 days active.
SearchDevices(ctx context.Context, query *SearchDeviceQuery) (*SearchDeviceQueryResult, error)
}
func ProvideAnonDBStore(sqlStore db.DB, deviceLimit int64) *AnonDBStore {
@@ -183,3 +210,64 @@ func (s *AnonDBStore) DeleteDevicesOlderThan(ctx context.Context, olderThan time
return err
}
func (s *AnonDBStore) SearchDevices(ctx context.Context, query *SearchDeviceQuery) (*SearchDeviceQueryResult, error) {
result := SearchDeviceQueryResult{
Devices: make([]*DeviceSearchHitDTO, 0),
}
err := s.sqlStore.WithDbSession(ctx, func(dbSess *db.Session) error {
if query.From.IsZero() && !query.To.IsZero() {
return fmt.Errorf("from date must be set if to date is set")
}
if !query.From.IsZero() && query.To.IsZero() {
return fmt.Errorf("to date must be set if from date is set")
}
// restricted only to last 30 days, if noting else specified
if query.From.IsZero() && query.To.IsZero() {
query.From = time.Now().Add(-anonymousDeviceExpiration)
query.To = time.Now()
}
sess := dbSess.Table("anon_device").Alias("d")
if query.Limit > 0 {
offset := query.Limit * (query.Page - 1)
sess.Limit(query.Limit, offset)
}
sess.Cols("d.id", "d.device_id", "d.client_ip", "d.user_agent", "d.updated_at")
if len(query.SortOpts) > 0 {
for i := range query.SortOpts {
for j := range query.SortOpts[i].Filter {
sess.OrderBy(query.SortOpts[i].Filter[j].OrderBy())
}
}
} else {
sess.Asc("d.user_agent")
}
// add to query about from and to session
sess.Where("d.updated_at BETWEEN ? AND ?", query.From.UTC(), query.To.UTC())
if query.Query != "" {
queryWithWildcards := "%" + strings.Replace(query.Query, "\\", "", -1) + "%"
sess.Where("d.client_ip "+s.sqlStore.GetDialect().LikeStr()+" ?", queryWithWildcards)
}
// get total
devices, err := s.ListDevices(ctx, &query.From, &query.To)
if err != nil {
return err
}
// cast to int64
result.TotalCount = int64(len(devices))
if err := sess.Find(&result.Devices); err != nil {
return err
}
result.Page = query.Page
result.PerPage = query.Limit
return nil
})
return &result, err
}
pkg/services/anonymous/anonimpl/anonstore/fake.go
+4
View File
@@ -19,3 +19,7 @@ func (s *FakeAnonStore) CreateOrUpdateDevice(ctx context.Context, device *Device
func (s *FakeAnonStore) CountDevices(ctx context.Context, from time.Time, to time.Time) (int64, error) {
return 0, nil
}
func (s *FakeAnonStore) SearchDevices(ctx context.Context, query SearchDeviceQuery) (*SearchDeviceQueryResult, error) {
return nil, nil
}
pkg/services/anonymous/anonimpl/api/api.go
+54
View File
@@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/anonymous/anonimpl/anonstore"
"github.com/grafana/grafana/pkg/services/anonymous/sortopts"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
@@ -50,6 +51,7 @@ func (api *AnonDeviceServiceAPI) RegisterAPIEndpoints() {
auth := accesscontrol.Middleware(api.accesscontrol)
api.RouterRegister.Group("/api/anonymous", func(anonRoutes routing.RouteRegister) {
anonRoutes.Get("/devices", auth(accesscontrol.EvalPermission(accesscontrol.ActionUsersRead)), routing.Wrap(api.ListDevices))
anonRoutes.Get("/search", auth(accesscontrol.EvalPermission(accesscontrol.ActionUsersRead)), routing.Wrap(api.SearchDevices))
})
}
@@ -89,8 +91,60 @@ func (api *AnonDeviceServiceAPI) ListDevices(c *contextmodel.ReqContext) respons
return response.JSON(http.StatusOK, resDevices)
}
// swagger:route POST /search devices SearchDevices
//
// # Lists all devices within the last 30 days
//
// Produces:
// - application/json
//
// Responses:
//
// 200: devicesSearchResponse
// 401: unauthorisedError
// 403: forbiddenError
// 404: notFoundError
// 500: internalServerError
func (api *AnonDeviceServiceAPI) SearchDevices(c *contextmodel.ReqContext) response.Response {
perPage := c.QueryInt("perpage")
if perPage <= 0 {
perPage = 100
}
page := c.QueryInt("page")
if page < 1 {
page = 1
}
searchQuery := c.Query("query")
sortOpts, err := sortopts.ParseSortQueryParam(c.Query("sort"))
if err != nil {
return response.ErrOrFallback(http.StatusInternalServerError, "Failed to list devices", err)
}
// TODO: potential add from and to time to query
query := &anonstore.SearchDeviceQuery{
Query: searchQuery,
Page: page,
Limit: perPage,
SortOpts: sortOpts,
}
results, err := api.store.SearchDevices(c.Req.Context(), query)
if err != nil {
return response.ErrOrFallback(http.StatusInternalServerError, "Failed to list devices", err)
}
return response.JSON(http.StatusOK, results)
}
// swagger:response devicesResponse
type DevicesResponse struct {
// in:body
Body []deviceDTO `json:"body"`
}
// swagger:response devicesSearchResponse
type DevicesSearchResponse struct {
// in:body
Body anonstore.SearchDeviceQueryResult `json:"body"`
}
pkg/services/anonymous/anonimpl/impl.go
+10
View File
@@ -154,6 +154,7 @@ func (a *AnonDeviceService) TagDevice(ctx context.Context, httpReq *http.Request
// ListDevices returns all devices that have been updated between the given times.
func (a *AnonDeviceService) ListDevices(ctx context.Context, from *time.Time, to *time.Time) ([]*anonstore.Device, error) {
if !a.cfg.AnonymousEnabled {
a.log.Debug("Anonymous access is disabled, returning empty result")
return []*anonstore.Device{}, nil
}
@@ -163,12 +164,21 @@ func (a *AnonDeviceService) ListDevices(ctx context.Context, from *time.Time, to
// CountDevices returns the number of devices that have been updated between the given times.
func (a *AnonDeviceService) CountDevices(ctx context.Context, from time.Time, to time.Time) (int64, error) {
if !a.cfg.AnonymousEnabled {
a.log.Debug("Anonymous access is disabled, returning empty result")
return 0, nil
}
return a.anonStore.CountDevices(ctx, from, to)
}
func (a *AnonDeviceService) SearchDevices(ctx context.Context, query *anonstore.SearchDeviceQuery) (*anonstore.SearchDeviceQueryResult, error) {
if !a.cfg.AnonymousEnabled {
a.log.Debug("Anonymous access is disabled, returning empty result")
return nil, nil
}
return a.anonStore.SearchDevices(ctx, query)
}
func (a *AnonDeviceService) Run(ctx context.Context) error {
ticker := time.NewTicker(2 * time.Hour)
pkg/services/anonymous/anonimpl/impl_test.go
+83
View File
@@ -177,3 +177,86 @@ func TestIntegrationAnonDeviceService_localCacheSafety(t *testing.T) {
assert.Equal(t, int64(0), stats["stats.anonymous.device.ui.count"].(int64))
}
func TestIntegrationDeviceService_SearchDevice(t *testing.T) {
testCases := []struct {
name string
insertDevices []*anonstore.Device
searchQuery anonstore.SearchDeviceQuery
expectedCount int
expectedDevice *anonstore.Device
}{
{
name: "two devices and limit set to 1",
insertDevices: []*anonstore.Device{
{
DeviceID: "32mdo31deeqwes",
ClientIP: "",
UserAgent: "test",
UpdatedAt: time.Now().UTC(),
},
{
DeviceID: "32mdo31deeqwes2",
ClientIP: "",
UserAgent: "test2",
UpdatedAt: time.Now().UTC(),
},
},
searchQuery: anonstore.SearchDeviceQuery{
Query: "",
Page: 1,
Limit: 1,
},
expectedCount: 1,
},
{
name: "two devices and search for client ip 192.1",
insertDevices: []*anonstore.Device{
{
DeviceID: "32mdo31deeqwes",
ClientIP: "192.168.0.2:10",
UserAgent: "",
UpdatedAt: time.Now().UTC(),
},
{
DeviceID: "32mdo31deeqwes2",
ClientIP: "192.268.1.3:200",
UserAgent: "",
UpdatedAt: time.Now().UTC(),
},
},
searchQuery: anonstore.SearchDeviceQuery{
Query: "192.1",
Page: 1,
Limit: 50,
},
expectedCount: 1,
expectedDevice: &anonstore.Device{
DeviceID: "32mdo31deeqwes",
ClientIP: "192.168.0.2:10",
UserAgent: "",
UpdatedAt: time.Now().UTC(),
},
},
}
store := db.InitTestDB(t)
anonService := ProvideAnonymousDeviceService(&usagestats.UsageStatsMock{},
&authntest.FakeService{}, store, setting.NewCfg(), orgtest.NewOrgServiceFake(), nil, actest.FakeAccessControl{}, &routing.RouteRegisterImpl{})
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
for _, device := range tc.insertDevices {
err := anonService.anonStore.CreateOrUpdateDevice(context.Background(), device)
require.NoError(t, err)
}
devices, err := anonService.anonStore.SearchDevices(context.Background(), &tc.searchQuery)
require.NoError(t, err)
require.Len(t, devices.Devices, tc.expectedCount)
if tc.expectedDevice != nil {
device := devices.Devices[0]
require.Equal(t, tc.expectedDevice.UserAgent, device.UserAgent)
}
})
}
}
pkg/services/anonymous/sortopts/sortopts.go
+97
View File
@@ -0,0 +1,97 @@
package sortopts
import (
"fmt"
"sort"
"strings"
"github.com/grafana/grafana/pkg/services/search/model"
"github.com/grafana/grafana/pkg/util/errutil"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
var (
// SortOptionsByQueryParam is a map to translate the "sort" query param values to SortOption(s)
SortOptionsByQueryParam = map[string]model.SortOption{
"userAgent-asc": newSortOption("user_agent", false, 0),
"userAgent-desc": newSortOption("user_agent", true, 0),
"updatedAt-asc": newTimeSortOption("updated_at", false, 1),
"updatedAt-desc": newTimeSortOption("updated_at", true, 2),
}
ErrorUnknownSortingOption = errutil.BadRequest("unknown sorting option")
)
type Sorter struct {
Field string
LowerCase bool
Descending bool
WithTableName bool
}
func (s Sorter) OrderBy() string {
orderBy := "anon_device."
if !s.WithTableName {
orderBy = ""
}
orderBy += s.Field
if s.LowerCase {
orderBy = fmt.Sprintf("LOWER(%v)", orderBy)
}
if s.Descending {
return orderBy + " DESC"
}
return orderBy + " ASC"
}
func newSortOption(field string, desc bool, index int) model.SortOption {
direction := "asc"
description := ("A-Z")
if desc {
direction = "desc"
description = ("Z-A")
}
return model.SortOption{
Name: fmt.Sprintf("%v-%v", field, direction),
DisplayName: fmt.Sprintf("%v (%v)", cases.Title(language.Und).String(field), description),
Description: fmt.Sprintf("Sort %v in an alphabetically %vending order", field, direction),
Index: index,
Filter: []model.SortOptionFilter{Sorter{Field: field, Descending: desc}},
}
}
func newTimeSortOption(field string, desc bool, index int) model.SortOption {
direction := "asc"
description := ("Oldest-Newest")
if desc {
direction = "desc"
description = ("Newest-Oldest")
}
return model.SortOption{
Name: fmt.Sprintf("%v-%v", field, direction),
DisplayName: fmt.Sprintf("%v (%v)", cases.Title(language.Und).String(field), description),
Description: fmt.Sprintf("Sort %v by time in an %vending order", field, direction),
Index: index,
Filter: []model.SortOptionFilter{Sorter{Field: field, Descending: desc}},
}
}
// ParseSortQueryParam parses the "sort" query param and returns an ordered list of SortOption(s)
func ParseSortQueryParam(param string) ([]model.SortOption, error) {
opts := []model.SortOption{}
if param != "" {
optsStr := strings.Split(param, ",")
for i := range optsStr {
if opt, ok := SortOptionsByQueryParam[optsStr[i]]; !ok {
return nil, ErrorUnknownSortingOption.Errorf("%v option unknown", optsStr[i])
} else {
opts = append(opts, opt)
}
}
sort.Slice(opts, func(i, j int) bool {
return opts[i].Index < opts[j].Index || (opts[i].Index == opts[j].Index && opts[i].Name < opts[j].Name)
})
}
return opts, nil
}