Auth: Add anonymous users view and stats (#78685)

* Add anonymous stats and user table

- anonymous users users page
- add feature toggle `anonymousAccess`
- remove check for enterprise for `Device-Id` header in request
- add anonusers/device count to stats

* promise all, review comments

* make use of promise all settled

* refactoring: devices instead of users

* review comments, moved countdevices to httpserver

* fakeAnonService for tests and generate openapi spec

* do not commit openapi3 and api-merged

* add openapi

* Apply suggestions from code review

Co-authored-by: Alex Khomenko <Clarity-89@users.noreply.github.com>

* formatin

* precise anon devices to avoid confusion

---------

Co-authored-by: Alex Khomenko <Clarity-89@users.noreply.github.com>
Co-authored-by: jguer <me@jguer.space>

pkg/api/admin.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"net/http"
+	"time"
 
 	"github.com/grafana/grafana/pkg/api/response"
 	ac "github.com/grafana/grafana/pkg/services/accesscontrol"
@@ -63,6 +64,12 @@ func (hs *HTTPServer) AdminGetStats(c *contextmodel.ReqContext) response.Respons
 	if err != nil {
 		return response.Error(500, "Failed to get admin stats from database", err)
 	}
+	thirtyDays := 30 * 24 * time.Hour
+	devicesCount, err := hs.anonService.CountDevices(c.Req.Context(), time.Now().Add(-thirtyDays), time.Now().Add(time.Minute))
+	if err != nil {
+		return response.Error(500, "Failed to get anon stats from database", err)
+	}
+	adminStats.AnonymousStats.ActiveDevices = devicesCount
 
 	return response.JSON(http.StatusOK, adminStats)
 }

pkg/api/admin_test.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/db/dbtest"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
+	"github.com/grafana/grafana/pkg/services/anonymous/anontest"
+	"github.com/grafana/grafana/pkg/services/stats"
 	"github.com/grafana/grafana/pkg/services/stats/statstest"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/web/webtest"
@@ -150,11 +152,16 @@ func TestAdmin_AccessControl(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.desc, func(t *testing.T) {
+			fakeStatsService := statstest.NewFakeService()
+			fakeStatsService.ExpectedAdminStats = &stats.AdminStats{}
+			fakeAnonService := anontest.NewFakeService()
+			fakeAnonService.ExpectedCountDevices = 0
 			server := SetupAPITestServer(t, func(hs *HTTPServer) {
 				hs.Cfg = setting.NewCfg()
 				hs.SQLStore = dbtest.NewFakeDB()
 				hs.SettingsProvider = &setting.OSSImpl{Cfg: hs.Cfg}
-				hs.statsService = statstest.NewFakeService()
+				hs.statsService = fakeStatsService
+				hs.anonService = fakeAnonService
 			})
 
 			res, err := server.Send(webtest.RequestWithSignedInUser(server.NewGetRequest(tt.url), userWithPermissions(1, tt.permissions)))

pkg/api/http_server.go

@@ -16,6 +16,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 
+	"github.com/grafana/grafana/pkg/services/anonymous"
 	grafanaapiserver "github.com/grafana/grafana/pkg/services/grafana-apiserver"
 	"github.com/grafana/grafana/pkg/services/grafana-apiserver/endpoints/request"
 
@@ -206,6 +207,7 @@ type HTTPServer struct {
 	promRegister         prometheus.Registerer
 	clientConfigProvider grafanaapiserver.DirectRestConfigProvider
 	namespacer           request.NamespaceMapper
+	anonService          anonymous.Service
 }
 
 type ServerOptions struct {
@@ -247,7 +249,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	accesscontrolService accesscontrol.Service, navTreeService navtree.Service,
 	annotationRepo annotations.Repository, tagService tag.Service, searchv2HTTPService searchV2.SearchHTTPService, oauthTokenService oauthtoken.OAuthTokenService,
 	statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service,
-	starApi *starApi.API, promRegister prometheus.Registerer, clientConfigProvider grafanaapiserver.DirectRestConfigProvider,
+	starApi *starApi.API, promRegister prometheus.Registerer, clientConfigProvider grafanaapiserver.DirectRestConfigProvider, anonService anonymous.Service,
 ) (*HTTPServer, error) {
 	web.Env = cfg.Env
 	m := web.New()
@@ -348,6 +350,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		promRegister:                 promRegister,
 		clientConfigProvider:         clientConfigProvider,
 		namespacer:                   request.GetNamespaceMapper(cfg),
+		anonService:                  anonService,
 	}
 	if hs.Listener != nil {
 		hs.log.Debug("Using provided listener")

pkg/services/anonymous/anonimpl/anonstore/database.go

@@ -21,11 +21,11 @@ type AnonDBStore struct {
 
 type Device struct {
 	ID        int64     `json:"-" xorm:"id" db:"id"`
-	DeviceID  string    `json:"device_id" xorm:"device_id" db:"device_id"`
-	ClientIP  string    `json:"client_ip" xorm:"client_ip" db:"client_ip"`
-	UserAgent string    `json:"user_agent" xorm:"user_agent" db:"user_agent"`
-	CreatedAt time.Time `json:"created_at" xorm:"created_at" db:"created_at"`
-	UpdatedAt time.Time `json:"updated_at" xorm:"updated_at" db:"updated_at"`
+	DeviceID  string    `json:"deviceId" xorm:"device_id" db:"device_id"`
+	ClientIP  string    `json:"clientIp" xorm:"client_ip" db:"client_ip"`
+	UserAgent string    `json:"userAgent" xorm:"user_agent" db:"user_agent"`
+	CreatedAt time.Time `json:"createdAt" xorm:"created_at" db:"created_at"`
+	UpdatedAt time.Time `json:"updatedAt" xorm:"updated_at" db:"updated_at"`
 }
 
 func (a *Device) CacheKey() string {

pkg/services/anonymous/anonimpl/api/api.go

@@ -0,0 +1,98 @@
+package api
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/grafana/grafana/pkg/api/dtos"
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/accesscontrol"
+	"github.com/grafana/grafana/pkg/services/anonymous/anonimpl/anonstore"
+	contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+const (
+	thirtyDays = 30 * 24 * time.Hour
+)
+
+type deviceDTO struct {
+	anonstore.Device
+	LastSeenAt string `json:"lastSeenAt"`
+	AvatarUrl  string `json:"avatarUrl"`
+}
+
+type AnonDeviceServiceAPI struct {
+	cfg            *setting.Cfg
+	store          anonstore.AnonStore
+	accesscontrol  accesscontrol.AccessControl
+	RouterRegister routing.RouteRegister
+	log            log.Logger
+}
+
+func NewAnonDeviceServiceAPI(
+	cfg *setting.Cfg,
+	anonstore anonstore.AnonStore,
+	accesscontrol accesscontrol.AccessControl,
+	routerRegister routing.RouteRegister,
+) *AnonDeviceServiceAPI {
+	return &AnonDeviceServiceAPI{
+		cfg:            cfg,
+		store:          anonstore,
+		accesscontrol:  accesscontrol,
+		RouterRegister: routerRegister,
+		log:            log.New("anon.api"),
+	}
+}
+
+func (api *AnonDeviceServiceAPI) RegisterAPIEndpoints() {
+	auth := accesscontrol.Middleware(api.accesscontrol)
+	api.RouterRegister.Group("/api/anonymous", func(anonRoutes routing.RouteRegister) {
+		anonRoutes.Get("/devices", auth(accesscontrol.EvalPermission(accesscontrol.ActionUsersRead)), routing.Wrap(api.ListDevices))
+	})
+}
+
+// swagger:route GET /stats devices listDevices
+//
+// # Lists all devices within the last 30 days
+//
+// Produces:
+// - application/json
+//
+// Responses:
+//
+//	200: devicesResponse
+//	401: unauthorisedError
+//	403: forbiddenError
+//	404: notFoundError
+//	500: internalServerError
+func (api *AnonDeviceServiceAPI) ListDevices(c *contextmodel.ReqContext) response.Response {
+	fromTime := time.Now().Add(-thirtyDays)
+	toTime := time.Now()
+
+	devices, err := api.store.ListDevices(c.Req.Context(), &fromTime, &toTime)
+	if err != nil {
+		return response.ErrOrFallback(http.StatusInternalServerError, "Failed to list devices", err)
+	}
+
+	// convert to response format
+	resDevices := make([]*deviceDTO, 0, len(devices))
+	for _, device := range devices {
+		resDevices = append(resDevices, &deviceDTO{
+			Device:     *device,
+			LastSeenAt: util.GetAgeString(device.UpdatedAt),
+			AvatarUrl:  dtos.GetGravatarUrl(device.DeviceID),
+		})
+	}
+
+	return response.JSON(http.StatusOK, resDevices)
+}
+
+// swagger:response devicesResponse
+type DevicesResponse struct {
+	// in:body
+	Body []deviceDTO `json:"body"`
+}

pkg/services/anonymous/anonimpl/client_test.go

@@ -49,7 +49,7 @@ func TestAnonymous_Authenticate(t *testing.T) {
 				cfg:               tt.cfg,
 				log:               log.NewNopLogger(),
 				orgService:        &orgtest.FakeOrgService{ExpectedOrg: tt.org, ExpectedError: tt.err},
-				anonDeviceService: &anontest.FakeAnonymousSessionService{},
+				anonDeviceService: anontest.NewFakeService(),
 			}
 
 			identity, err := c.Authenticate(context.Background(), &authn.Request{})

pkg/services/anonymous/anonimpl/impl.go

@@ -5,13 +5,16 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/infra/localcache"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/network"
 	"github.com/grafana/grafana/pkg/infra/serverlock"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
+	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/anonymous"
 	"github.com/grafana/grafana/pkg/services/anonymous/anonimpl/anonstore"
+	"github.com/grafana/grafana/pkg/services/anonymous/anonimpl/api"
 	"github.com/grafana/grafana/pkg/services/authn"
 	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/setting"
@@ -31,7 +34,7 @@ type AnonDeviceService struct {
 
 func ProvideAnonymousDeviceService(usageStats usagestats.Service, authBroker authn.Service,
 	anonStore anonstore.AnonStore, cfg *setting.Cfg, orgService org.Service,
-	serverLockService *serverlock.ServerLockService,
+	serverLockService *serverlock.ServerLockService, accesscontrol accesscontrol.AccessControl, routeRegister routing.RouteRegister,
 ) *AnonDeviceService {
 	a := &AnonDeviceService{
 		log:        log.New("anonymous-session-service"),
@@ -54,6 +57,9 @@ func ProvideAnonymousDeviceService(usageStats usagestats.Service, authBroker aut
 		authBroker.RegisterPostLoginHook(a.untagDevice, 100)
 	}
 
+	anonAPI := api.NewAnonDeviceServiceAPI(cfg, anonStore, accesscontrol, routeRegister)
+	anonAPI.RegisterAPIEndpoints()
+
 	return a
 }
 
@@ -142,6 +148,16 @@ func (a *AnonDeviceService) TagDevice(ctx context.Context, httpReq *http.Request
 	return nil
 }
 
+// ListDevices returns all devices that have been updated between the given times.
+func (a *AnonDeviceService) ListDevices(ctx context.Context, from *time.Time, to *time.Time) ([]*anonstore.Device, error) {
+	return a.anonStore.ListDevices(ctx, from, to)
+}
+
+// CountDevices returns the number of devices that have been updated between the given times.
+func (a *AnonDeviceService) CountDevices(ctx context.Context, from time.Time, to time.Time) (int64, error) {
+	return a.anonStore.CountDevices(ctx, from, to)
+}
+
 func (a *AnonDeviceService) Run(ctx context.Context) error {
 	ticker := time.NewTicker(2 * time.Hour)
 

pkg/services/anonymous/anonimpl/impl_test.go

@@ -9,8 +9,10 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/usagestats"
+	"github.com/grafana/grafana/pkg/services/accesscontrol/actest"
 	"github.com/grafana/grafana/pkg/services/anonymous"
 	"github.com/grafana/grafana/pkg/services/anonymous/anonimpl/anonstore"
 	"github.com/grafana/grafana/pkg/services/authn/authntest"
@@ -113,7 +115,7 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
 			store := db.InitTestDB(t)
 			anonDBStore := anonstore.ProvideAnonDBStore(store)
 			anonService := ProvideAnonymousDeviceService(&usagestats.UsageStatsMock{},
-				&authntest.FakeService{}, anonDBStore, setting.NewCfg(), orgtest.NewOrgServiceFake(), nil)
+				&authntest.FakeService{}, anonDBStore, setting.NewCfg(), orgtest.NewOrgServiceFake(), nil, actest.FakeAccessControl{}, &routing.RouteRegisterImpl{})
 
 			for _, req := range tc.req {
 				err := anonService.TagDevice(context.Background(), req.httpReq, req.kind)
@@ -149,7 +151,7 @@ func TestIntegrationAnonDeviceService_localCacheSafety(t *testing.T) {
 	store := db.InitTestDB(t)
 	anonDBStore := anonstore.ProvideAnonDBStore(store)
 	anonService := ProvideAnonymousDeviceService(&usagestats.UsageStatsMock{},
-		&authntest.FakeService{}, anonDBStore, setting.NewCfg(), orgtest.NewOrgServiceFake(), nil)
+		&authntest.FakeService{}, anonDBStore, setting.NewCfg(), orgtest.NewOrgServiceFake(), nil, actest.FakeAccessControl{}, &routing.RouteRegisterImpl{})
 
 	req := &http.Request{
 		Header: http.Header{

pkg/services/anonymous/anontest/fake.go

@@ -3,13 +3,27 @@ package anontest
 import (
 	"context"
 	"net/http"
+	"time"
 
 	"github.com/grafana/grafana/pkg/services/anonymous"
 )
 
+type FakeService struct {
+	ExpectedCountDevices int64
+	ExpectedError        error
+}
+
+func NewFakeService() *FakeService {
+	return &FakeService{}
+}
+
 type FakeAnonymousSessionService struct {
 }
 
-func (f *FakeAnonymousSessionService) TagDevice(ctx context.Context, httpReq *http.Request, kind anonymous.DeviceKind) error {
+func (f *FakeService) TagDevice(ctx context.Context, httpReq *http.Request, kind anonymous.DeviceKind) error {
 	return nil
 }
+
+func (f *FakeService) CountDevices(ctx context.Context, from time.Time, to time.Time) (int64, error) {
+	return f.ExpectedCountDevices, nil
+}

pkg/services/anonymous/service.go

@@ -3,6 +3,7 @@ package anonymous
 import (
 	"context"
 	"net/http"
+	"time"
 )
 
 type DeviceKind string
@@ -13,4 +14,5 @@ const (
 
 type Service interface {
 	TagDevice(context.Context, *http.Request, DeviceKind) error
+	CountDevices(ctx context.Context, from time.Time, to time.Time) (int64, error)
 }

pkg/services/featuremgmt/registry.go

@@ -1082,6 +1082,13 @@ var (
 			FrontendOnly: true,
 			Owner:        grafanaBiSquad,
 		},
+		{
+			Name:         "displayAnonymousStats",
+			Description:  "Enables anonymous stats to be shown in the UI for Grafana",
+			Stage:        FeatureStageExperimental,
+			FrontendOnly: true,
+			Owner:        identityAccessTeam,
+		},
 	}
 )
 

pkg/services/featuremgmt/toggles_gen.csv

@@ -146,3 +146,4 @@ alertingSimplifiedRouting,experimental,@grafana/alerting-squad,false,false,false
 logRowsPopoverMenu,experimental,@grafana/observability-logs,false,false,false,true
 pluginsSkipHostEnvVars,experimental,@grafana/plugins-platform-backend,false,false,false,false
 regressionTransformation,experimental,@grafana/grafana-bi-squad,false,false,false,true
+displayAnonymousStats,experimental,@grafana/identity-access-team,false,false,false,true

pkg/services/featuremgmt/toggles_gen.go

@@ -594,4 +594,8 @@ const (
 	// FlagRegressionTransformation
 	// Enables regression analysis transformation
 	FlagRegressionTransformation = "regressionTransformation"
+
+	// FlagDisplayAnonymousStats
+	// Enables anonymous stats to be shown in the UI for Grafana
+	FlagDisplayAnonymousStats = "displayAnonymousStats"
 )

pkg/services/stats/models.go

@@ -77,6 +77,9 @@ type NotifierUsageStats struct {
 
 type GetAlertNotifierUsageStatsQuery struct{}
 
+type AnonymousStats struct {
+	ActiveDevices int64 `json:"activeDevices"`
+}
 type AdminStats struct {
 	Orgs                int64 `json:"orgs"`
 	Dashboards          int64 `json:"dashboards"`
@@ -101,6 +104,7 @@ type AdminStats struct {
 	DailyActiveViewers  int64 `json:"dailyActiveViewers"`
 	DailyActiveSessions int64 `json:"dailyActiveSessions"`
 	MonthlyActiveUsers  int64 `json:"monthlyActiveUsers"`
+	AnonymousStats
 }
 
 type GetAdminStatsQuery struct{}

pkg/services/stats/statstest/stats.go

@@ -7,6 +7,7 @@ import (
 )
 
 type FakeService struct {
+	ExpectedAdminStats             *stats.AdminStats
 	ExpectedSystemStats            *stats.SystemStats
 	ExpectedDataSourceStats        []*stats.DataSourceStats
 	ExpectedDataSourcesAccessStats []*stats.DataSourceAccessStats
@@ -20,7 +21,7 @@ func NewFakeService() *FakeService {
 }
 
 func (s *FakeService) GetAdminStats(ctx context.Context, query *stats.GetAdminStatsQuery) (*stats.AdminStats, error) {
-	return nil, s.ExpectedError
+	return s.ExpectedAdminStats, s.ExpectedError
 }
 
 func (s *FakeService) GetAlertNotifiersUsageStats(ctx context.Context, query *stats.GetAlertNotifierUsageStatsQuery) ([]*stats.NotifierUsageStats, error) {