AuthN: Embed an OAuth2 server for external service authentication (#68086)

* Moving POC files from #64283 to a new branch

Co-authored-by: Mihály Gyöngyösi <mgyongyosi@users.noreply.github.com>

* Adding missing permission definition

Co-authored-by: Mihály Gyöngyösi <mgyongyosi@users.noreply.github.com>

* Force the service instantiation while client isn't merged

Co-authored-by: Mihály Gyöngyösi <mgyongyosi@users.noreply.github.com>

* Merge conf with main

Co-authored-by: Mihály Gyöngyösi <mgyongyosi@users.noreply.github.com>

* Leave go-sqlite3 version unchanged

Co-authored-by: Mihály Gyöngyösi <mgyongyosi@users.noreply.github.com>

* tidy

Co-authored-by: Mihály Gyöngyösi <mgyongyosi@users.noreply.github.com>

* User SearchUserPermissions instead of SearchUsersPermissions

* Replace DummyKeyService with signingkeys.Service

* Use user🆔<id> as subject

* Fix introspection endpoint issue

* Add X-Grafana-Org-Id to get_resources.bash script

* Regenerate toggles_gen.go
* Fix basic.go

* Add GetExternalService tests

* Add GetPublicKeyScopes tests

* Add GetScopesOnUser tests

* Add GetScopes tests

* Add ParsePublicKeyPem tests

* Add database test for GetByName

* re-add comments

* client tests added

* Add GetExternalServicePublicKey tests

* Add other test case to GetExternalServicePublicKey

* client_credentials grant test

* Add test to jwtbearer grant

* Test Comments

* Add handleKeyOptions tests

* Add RSA key generation test

* Add ECDSA by default to EmbeddedSigningKeysService

* Clean up org id scope and audiences

* Add audiences to the DB

* Fix check on Audience

* Fix double import

* Add AC Store mock and align oauthserver tests

* Fix test after rebase

* Adding missing store function to mock

* Fix double import

* Add CODEOWNER

* Fix some linting errors

* errors don't need type assertion

* Typo codeowners

* use mockery for oauthserver store

* Add feature toggle check

* Fix db tests to handle the feature flag

* Adding call to DeleteExternalServiceRole

* Fix flaky test

* Re-organize routes comments and plan futur work

* Add client_id check to Extended JWT client

* Clean up

* Fix

* Remove background service registry instantiation of the OAuth server

* Comment cleanup

* Remove unused client function

* Update go.mod to use the latest ory/fosite commit

* Remove oauth2_server related configs from defaults.ini

* Add audiences to DTO

* Fix flaky test

* Remove registration endpoint and demo scripts. Document code

* Rename packages

* Remove the OAuthService vs OAuthServer confusion

* fix incorrect import ext_jwt_test

* Comments and order

* Comment basic auth

* Remove unecessary todo

* Clean api

* Moving ParsePublicKeyPem to utils

* re ordering functions in service.go

* Fix comment

* comment on the redirect uri

* Add RBAC actions, not only scopes

* Fix tests

* re-import featuremgmt in migrations

* Fix wire

* Fix scopes in test

* Fix flaky test

* Remove todo, the intersection should always return the minimal set

* Remove unecessary check from intersection code

* Allow env overrides on settings

* remove the term app name

* Remove app keyword for client instead and use Name instead of ExternalServiceName

* LogID remove ExternalService ref

* Use Name instead of ExternalServiceName

* Imports order

* Inline

* Using ExternalService and ExternalServiceDTO

* Remove xorm tags

* comment

* Rename client files

* client -> external service

* comments

* Move test to correct package

* slimmer test

* cachedUser -> cachedExternalService

* Fix aggregate store test

* PluginAuthSession -> AuthSession

* Revert the nil cehcks

* Remove unecessary extra

* Removing custom session

* fix typo in test

* Use constants for tests

* Simplify HandleToken tests

* Refactor the HandleTokenRequest test

* test message

* Review test

* Prevent flacky test on client as well

* go imports

* Revert changes from 526e48ad45

* AuthN: Change the External Service registration form (#68649)

* AuthN: change the External Service registration form

* Gen default permissions

* Change demo script registration form

* Remove unecessary comment

* Nit.

* Reduce cyclomatic complexity

* Remove demo_scripts

* Handle case with no service account

* Comments

* Group key gen

* Nit.

* Check the SaveExternalService test

* Rename cachedUser to cachedClient in test

* One more test case to database test

* Comments

* Remove last org scope

Co-authored-by: Mihály Gyöngyösi <mgyongyosi@users.noreply.github.com>

* Update pkg/services/oauthserver/utils/utils_test.go

* Update pkg/services/sqlstore/migrations/oauthserver/migrations.go

Remove comment

* Update pkg/setting/setting.go

Co-authored-by: Gabriel MABILLE <gamab@users.noreply.github.com>

---------

Co-authored-by: Mihály Gyöngyösi <mgyongyosi@users.noreply.github.com>

pkg/services/authn/authnimpl/service.go

@@ -27,6 +27,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ldap/service"
 	"github.com/grafana/grafana/pkg/services/login"
 	"github.com/grafana/grafana/pkg/services/loginattempt"
+	"github.com/grafana/grafana/pkg/services/oauthserver"
 	"github.com/grafana/grafana/pkg/services/oauthtoken"
 	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/services/quota"
@@ -64,7 +65,7 @@ func ProvideService(
 	features *featuremgmt.FeatureManager, oauthTokenService oauthtoken.OAuthTokenService,
 	socialService social.Service, cache *remotecache.RemoteCache,
 	ldapService service.LDAP, registerer prometheus.Registerer,
-	signingKeysService signingkeys.Service,
+	signingKeysService signingkeys.Service, oauthServer oauthserver.OAuth2Server,
 ) authn.Service {
 	s := &Service{
 		log:            log.New("authn.service"),
@@ -131,7 +132,7 @@ func ProvideService(
 	}
 
 	if s.cfg.ExtendedJWTAuthEnabled && features.IsEnabled(featuremgmt.FlagExternalServiceAuth) {
-		s.RegisterClient(clients.ProvideExtendedJWT(userService, cfg, signingKeysService))
+		s.RegisterClient(clients.ProvideExtendedJWT(userService, cfg, signingKeysService, oauthServer))
 	}
 
 	for name := range socialService.GetOAuthProviders() {

pkg/services/authn/clients/basic.go

@@ -2,6 +2,7 @@ package clients
 
 import (
 	"context"
+	"strings"
 
 	"github.com/grafana/grafana/pkg/services/authn"
 	"github.com/grafana/grafana/pkg/util/errutil"
@@ -39,6 +40,13 @@ func (c *Basic) Authenticate(ctx context.Context, r *authn.Request) (*authn.Iden
 }
 
 func (c *Basic) Test(ctx context.Context, r *authn.Request) bool {
+	if r.HTTPRequest == nil {
+		return false
+	}
+	// The OAuth2 introspection endpoint uses basic auth but is handled by the oauthserver package.
+	if strings.HasPrefix(r.HTTPRequest.RequestURI, "/oauth2/introspect") {
+		return false
+	}
 	return looksLikeBasicAuthRequest(r)
 }
 

pkg/services/authn/clients/ext_jwt.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/authn"
+	"github.com/grafana/grafana/pkg/services/oauthserver"
 	"github.com/grafana/grafana/pkg/services/signingkeys"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
@@ -30,12 +31,13 @@ const (
 	rfc9068MediaType      = "application/at+jwt"
 )
 
-func ProvideExtendedJWT(userService user.Service, cfg *setting.Cfg, signingKeys signingkeys.Service) *ExtendedJWT {
+func ProvideExtendedJWT(userService user.Service, cfg *setting.Cfg, signingKeys signingkeys.Service, oauthServer oauthserver.OAuth2Server) *ExtendedJWT {
 	return &ExtendedJWT{
 		cfg:         cfg,
 		log:         log.New(authn.ClientExtendedJWT),
 		userService: userService,
 		signingKeys: signingKeys,
+		oauthServer: oauthServer,
 	}
 }
 
@@ -44,6 +46,7 @@ type ExtendedJWT struct {
 	log         log.Logger
 	userService user.Service
 	signingKeys signingkeys.Service
+	oauthServer oauthserver.OAuth2Server
 }
 
 type ExtendedJWTClaims struct {
@@ -211,10 +214,9 @@ func (s *ExtendedJWT) validateClientIdClaim(ctx context.Context, claims Extended
 		return fmt.Errorf("missing 'client_id' claim")
 	}
 
-	// TODO: Implement the validation for client_id when the OAuth server is ready.
-	// if _, err := s.oauthService.GetExternalService(ctx, clientId); err != nil {
-	// 	return fmt.Errorf("invalid 'client_id' claim: %s", clientIdClaim)
-	// }
+	if _, err := s.oauthServer.GetExternalService(ctx, claims.ClientID); err != nil {
+		return fmt.Errorf("invalid 'client_id' claim: %s", claims.ClientID)
+	}
 
 	return nil
 }

pkg/services/authn/clients/ext_jwt_test.go

@@ -15,6 +15,8 @@ import (
 	"github.com/grafana/grafana/pkg/models/roletype"
 	"github.com/grafana/grafana/pkg/services/authn"
 	"github.com/grafana/grafana/pkg/services/login"
+	"github.com/grafana/grafana/pkg/services/oauthserver"
+	"github.com/grafana/grafana/pkg/services/oauthserver/oastest"
 	"github.com/grafana/grafana/pkg/services/signingkeys/signingkeystest"
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/services/user/usertest"
@@ -49,7 +51,7 @@ var (
 	pk, _ = rsa.GenerateKey(rand.Reader, 4096)
 )
 
-func TestExtendedJWTTest(t *testing.T) {
+func TestExtendedJWT_Test(t *testing.T) {
 	type testCase struct {
 		name           string
 		cfg            *setting.Cfg
@@ -105,7 +107,7 @@ func TestExtendedJWTTest(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			extJwtClient := setupTestCtx(t, nil, tc.cfg)
+			env := setupTestCtx(t, tc.cfg)
 
 			validHTTPReq := &http.Request{
 				Header: map[string][]string{
@@ -113,7 +115,7 @@ func TestExtendedJWTTest(t *testing.T) {
 				},
 			}
 
-			actual := extJwtClient.Test(context.Background(), &authn.Request{
+			actual := env.s.Test(context.Background(), &authn.Request{
 				HTTPRequest: validHTTPReq,
 				Resp:        nil,
 			})
@@ -123,22 +125,22 @@ func TestExtendedJWTTest(t *testing.T) {
 	}
 }
 
-func TestExtendedJWTAuthenticate(t *testing.T) {
+func TestExtendedJWT_Authenticate(t *testing.T) {
 	type testCase struct {
-		name         string
-		payload      ExtendedJWTClaims
-		orgID        int64
-		want         *authn.Identity
-		userSvcSetup func(userSvc *usertest.FakeUserService)
-		wantErr      bool
+		name        string
+		payload     ExtendedJWTClaims
+		orgID       int64
+		want        *authn.Identity
+		initTestEnv func(env *testEnv)
+		wantErr     bool
 	}
 	testCases := []testCase{
 		{
 			name:    "successful authentication",
 			payload: validPayload,
 			orgID:   1,
-			userSvcSetup: func(userSvc *usertest.FakeUserService) {
-				userSvc.ExpectedSignedInUser = &user.SignedInUser{
+			initTestEnv: func(env *testEnv) {
+				env.userSvc.ExpectedSignedInUser = &user.SignedInUser{
 					UserID:  2,
 					OrgID:   1,
 					OrgRole: roletype.RoleAdmin,
@@ -242,8 +244,8 @@ func TestExtendedJWTAuthenticate(t *testing.T) {
 			},
 			orgID: 1,
 			want:  nil,
-			userSvcSetup: func(userSvc *usertest.FakeUserService) {
-				userSvc.ExpectedError = user.ErrUserNotFound
+			initTestEnv: func(env *testEnv) {
+				env.userSvc.ExpectedError = user.ErrUserNotFound
 			},
 			wantErr: true,
 		},
@@ -265,33 +267,34 @@ func TestExtendedJWTAuthenticate(t *testing.T) {
 			want:    nil,
 			wantErr: true,
 		},
-		// {
-		// 	name: "should return error when the entitlements are not in the correct format",
-		// 	payload: ExtendedJWTClaims{
-		// 		Claims: jwt.Claims{
-		// 			Issuer:   "http://localhost:3000",
-		// 			Subject:  "user:id:2",
-		// 			Audience: jwt.Audience{"http://localhost:3000"},
-		// 			ID:       "1234567890",
-		// 			Expiry:   jwt.NewNumericDate(time.Date(2023, 5, 3, 0, 0, 0, 0, time.UTC)),
-		// 			IssuedAt: jwt.NewNumericDate(time.Date(2023, 5, 2, 0, 0, 0, 0, time.UTC)),
-		// 		},
-		// 		ClientID:     "grafana",
-		// 		Scopes:       []string{"profile", "groups"},
-		// 		Entitlements: []string{"dashboards:create", "folders:read"},
-		// 	},
-		// 	orgID:   1,
-		// 	want:    nil,
-		// 	wantErr: true,
-		// },
+		{
+			name: "should return error when the client was not found",
+			payload: ExtendedJWTClaims{
+				Claims: jwt.Claims{
+					Issuer:   "http://localhost:3000",
+					Subject:  "user:id:2",
+					Audience: jwt.Audience{"http://localhost:3000"},
+					ID:       "1234567890",
+					Expiry:   jwt.NewNumericDate(time.Date(2023, 5, 3, 0, 0, 0, 0, time.UTC)),
+					IssuedAt: jwt.NewNumericDate(time.Date(2023, 5, 2, 0, 0, 0, 0, time.UTC)),
+				},
+				ClientID: "unknown-client-id",
+				Scopes:   []string{"profile", "groups"},
+			},
+			initTestEnv: func(env *testEnv) {
+				env.oauthSvc.ExpectedErr = oauthserver.ErrClientNotFound("unknown-client-id")
+			},
+			orgID:   1,
+			want:    nil,
+			wantErr: true,
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			userSvc := &usertest.FakeUserService{}
-			extJwtClient := setupTestCtx(t, userSvc, nil)
-			if tc.userSvcSetup != nil {
-				tc.userSvcSetup(userSvc)
+			env := setupTestCtx(t, nil)
+			if tc.initTestEnv != nil {
+				tc.initTestEnv(env)
 			}
 
 			validHTTPReq := &http.Request{
@@ -302,7 +305,7 @@ func TestExtendedJWTAuthenticate(t *testing.T) {
 
 			mockTimeNow(time.Date(2023, 5, 2, 0, 1, 0, 0, time.UTC))
 
-			id, err := extJwtClient.Authenticate(context.Background(), &authn.Request{
+			id, err := env.s.Authenticate(context.Background(), &authn.Request{
 				OrgID:       tc.orgID,
 				HTTPRequest: validHTTPReq,
 				Resp:        nil,
@@ -487,7 +490,7 @@ func TestVerifyRFC9068TokenFailureScenarios(t *testing.T) {
 		},
 	}
 
-	extJwtClient := setupTestCtx(t, nil, nil)
+	env := setupTestCtx(t, nil)
 	mockTimeNow(time.Date(2023, 5, 2, 0, 1, 0, 0, time.UTC))
 
 	for _, tc := range testCases {
@@ -496,13 +499,13 @@ func TestVerifyRFC9068TokenFailureScenarios(t *testing.T) {
 				tc.alg = jose.RS256
 			}
 			tokenToTest := generateToken(tc.payload, pk, tc.alg)
-			_, err := extJwtClient.verifyRFC9068Token(context.Background(), tokenToTest)
+			_, err := env.s.verifyRFC9068Token(context.Background(), tokenToTest)
 			require.Error(t, err)
 		})
 	}
 }
 
-func setupTestCtx(t *testing.T, userSvc user.Service, cfg *setting.Cfg) *ExtendedJWT {
+func setupTestCtx(t *testing.T, cfg *setting.Cfg) *testEnv {
 	if cfg == nil {
 		cfg = &setting.Cfg{
 			ExtendedJWTAuthEnabled:    true,
@@ -514,8 +517,22 @@ func setupTestCtx(t *testing.T, userSvc user.Service, cfg *setting.Cfg) *Extende
 	signingKeysSvc := &signingkeystest.FakeSigningKeysService{}
 	signingKeysSvc.ExpectedServerPublicKey = &pk.PublicKey
 
-	extJwtClient := ProvideExtendedJWT(userSvc, cfg, signingKeysSvc)
-	return extJwtClient
+	userSvc := &usertest.FakeUserService{}
+	oauthSvc := &oastest.FakeService{}
+
+	extJwtClient := ProvideExtendedJWT(userSvc, cfg, signingKeysSvc, oauthSvc)
+
+	return &testEnv{
+		oauthSvc: oauthSvc,
+		userSvc:  userSvc,
+		s:        extJwtClient,
+	}
+}
+
+type testEnv struct {
+	oauthSvc *oastest.FakeService
+	userSvc  *usertest.FakeUserService
+	s        *ExtendedJWT
 }
 
 func generateToken(payload ExtendedJWTClaims, signingKey interface{}, alg jose.SignatureAlgorithm) string {