Compare commits

...

1 Commits

Author SHA1 Message Date
Roberto Jimenez Sanchez a6a9358d5b Provisioning: Implement connection repositories endpoint for GitHub
This change implements the `/repositories` subresource endpoint for Connection
resources, enabling listing of repositories accessible through a GitHub App
connection.

Changes:
- Add ListRepositories method to Connection interface
- Add ListInstallationRepositories to GitHub Client interface
- Implement GitHub client method to list installation repositories
  - Creates installation access token from JWT
  - Handles pagination up to 1000 repos
- Implement ListRepositories in GitHub Connection
- Update connectionRepositoriesConnector to use Connection.ListRepositories
- Add ConnectionGetter interface and GetConnection method to APIBuilder
- Add comprehensive tests for the new functionality
2026-01-13 09:56:40 +01:00
10 changed files with 478 additions and 19 deletions
@@ -2,6 +2,8 @@ package connection
import (
"context"
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
)
//go:generate mockery --name Connection --structname MockConnection --inpackage --filename connection_mock.go --with-expecter
@@ -13,4 +15,8 @@ type Connection interface {
// Mutate performs in place mutation of the underneath resource.
Mutate(context.Context) error
// ListRepositories returns the list of repositories accessible through this connection.
// The repositories returned are external repositories from the git provider (e.g., GitHub, GitLab).
ListRepositories(ctx context.Context) ([]provisioning.ExternalRepository, error)
}
@@ -5,6 +5,7 @@ package connection
import (
context "context"
v0alpha1 "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
mock "github.com/stretchr/testify/mock"
)
@@ -21,6 +22,64 @@ func (_m *MockConnection) EXPECT() *MockConnection_Expecter {
return &MockConnection_Expecter{mock: &_m.Mock}
}
// ListRepositories provides a mock function with given fields: ctx
func (_m *MockConnection) ListRepositories(ctx context.Context) ([]v0alpha1.ExternalRepository, error) {
ret := _m.Called(ctx)
if len(ret) == 0 {
panic("no return value specified for ListRepositories")
}
var r0 []v0alpha1.ExternalRepository
var r1 error
if rf, ok := ret.Get(0).(func(context.Context) ([]v0alpha1.ExternalRepository, error)); ok {
return rf(ctx)
}
if rf, ok := ret.Get(0).(func(context.Context) []v0alpha1.ExternalRepository); ok {
r0 = rf(ctx)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]v0alpha1.ExternalRepository)
}
}
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
r1 = rf(ctx)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockConnection_ListRepositories_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListRepositories'
type MockConnection_ListRepositories_Call struct {
*mock.Call
}
// ListRepositories is a helper method to define mock.On call
// - ctx context.Context
func (_e *MockConnection_Expecter) ListRepositories(ctx interface{}) *MockConnection_ListRepositories_Call {
return &MockConnection_ListRepositories_Call{Call: _e.mock.On("ListRepositories", ctx)}
}
func (_c *MockConnection_ListRepositories_Call) Run(run func(ctx context.Context)) *MockConnection_ListRepositories_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context))
})
return _c
}
func (_c *MockConnection_ListRepositories_Call) Return(_a0 []v0alpha1.ExternalRepository, _a1 error) *MockConnection_ListRepositories_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockConnection_ListRepositories_Call) RunAndReturn(run func(context.Context) ([]v0alpha1.ExternalRepository, error)) *MockConnection_ListRepositories_Call {
_c.Call.Return(run)
return _c
}
// Mutate provides a mock function with given fields: _a0
func (_m *MockConnection) Mutate(_a0 context.Context) error {
ret := _m.Called(_a0)
@@ -22,6 +22,19 @@ type Client interface {
// Apps and installations
GetApp(ctx context.Context) (App, error)
GetAppInstallation(ctx context.Context, installationID string) (AppInstallation, error)
// Repositories
ListInstallationRepositories(ctx context.Context, installationID string) ([]Repository, error)
}
// Repository represents a GitHub repository accessible through an installation.
type Repository struct {
// Name of the repository
Name string
// Owner is the user or organization that owns the repository
Owner string
// URL of the repository (HTML URL)
URL string
}
// App represents a Github App.
@@ -91,3 +104,68 @@ func (r *githubClient) GetAppInstallation(ctx context.Context, installationID st
Enabled: installation.GetSuspendedAt().IsZero(),
}, nil
}
const (
maxRepositories = 1000 // Maximum number of repositories to fetch
)
// ListInstallationRepositories lists all repositories accessible by the specified GitHub App installation.
// It first creates an installation access token using the JWT, then uses that token to list repositories.
func (r *githubClient) ListInstallationRepositories(ctx context.Context, installationID string) ([]Repository, error) {
id, err := strconv.ParseInt(installationID, 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid installation ID: %s", installationID)
}
// Create an installation access token
installationToken, _, err := r.gh.Apps.CreateInstallationToken(ctx, id, nil)
if err != nil {
var ghErr *github.ErrorResponse
if errors.As(err, &ghErr) && ghErr.Response.StatusCode == http.StatusServiceUnavailable {
return nil, ErrServiceUnavailable
}
return nil, fmt.Errorf("create installation token: %w", err)
}
// Create a new client with the installation token
tokenClient := github.NewClient(nil).WithAuthToken(installationToken.GetToken())
var allRepos []Repository
opts := &github.ListOptions{
Page: 1,
PerPage: 100,
}
for {
result, resp, err := tokenClient.Apps.ListRepos(ctx, opts)
if err != nil {
var ghErr *github.ErrorResponse
if errors.As(err, &ghErr) && ghErr.Response.StatusCode == http.StatusServiceUnavailable {
return nil, ErrServiceUnavailable
}
return nil, fmt.Errorf("list repositories: %w", err)
}
for _, repo := range result.Repositories {
allRepos = append(allRepos, Repository{
Name: repo.GetName(),
Owner: repo.GetOwner().GetLogin(),
URL: repo.GetHTMLURL(),
})
}
// Check if we've exceeded the maximum allowed repositories
if len(allRepos) > maxRepositories {
return nil, fmt.Errorf("too many repositories to fetch (more than %d)", maxRepositories)
}
// If there are no more pages, break
if resp.NextPage == 0 {
break
}
opts.Page = resp.NextPage
}
return allRepos, nil
}
@@ -134,6 +134,65 @@ func (_c *MockClient_GetAppInstallation_Call) RunAndReturn(run func(context.Cont
return _c
}
// ListInstallationRepositories provides a mock function with given fields: ctx, installationID
func (_m *MockClient) ListInstallationRepositories(ctx context.Context, installationID string) ([]Repository, error) {
ret := _m.Called(ctx, installationID)
if len(ret) == 0 {
panic("no return value specified for ListInstallationRepositories")
}
var r0 []Repository
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) ([]Repository, error)); ok {
return rf(ctx, installationID)
}
if rf, ok := ret.Get(0).(func(context.Context, string) []Repository); ok {
r0 = rf(ctx, installationID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]Repository)
}
}
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
r1 = rf(ctx, installationID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockClient_ListInstallationRepositories_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListInstallationRepositories'
type MockClient_ListInstallationRepositories_Call struct {
*mock.Call
}
// ListInstallationRepositories is a helper method to define mock.On call
// - ctx context.Context
// - installationID string
func (_e *MockClient_Expecter) ListInstallationRepositories(ctx interface{}, installationID interface{}) *MockClient_ListInstallationRepositories_Call {
return &MockClient_ListInstallationRepositories_Call{Call: _e.mock.On("ListInstallationRepositories", ctx, installationID)}
}
func (_c *MockClient_ListInstallationRepositories_Call) Run(run func(ctx context.Context, installationID string)) *MockClient_ListInstallationRepositories_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string))
})
return _c
}
func (_c *MockClient_ListInstallationRepositories_Call) Return(_a0 []Repository, _a1 error) *MockClient_ListInstallationRepositories_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockClient_ListInstallationRepositories_Call) RunAndReturn(run func(context.Context, string) ([]Repository, error)) *MockClient_ListInstallationRepositories_Call {
_c.Call.Return(run)
return _c
}
// NewMockClient creates a new instance of MockClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockClient(t interface {
@@ -187,6 +187,31 @@ func toError(name string, list field.ErrorList) error {
)
}
// ListRepositories returns the list of repositories accessible through this GitHub App connection.
func (c *Connection) ListRepositories(ctx context.Context) ([]provisioning.ExternalRepository, error) {
if c.obj.Spec.GitHub == nil {
return nil, fmt.Errorf("github configuration is required")
}
ghClient := c.ghFactory.New(ctx, c.obj.Secure.Token.Create)
repos, err := ghClient.ListInstallationRepositories(ctx, c.obj.Spec.GitHub.InstallationID)
if err != nil {
return nil, fmt.Errorf("list installation repositories: %w", err)
}
result := make([]provisioning.ExternalRepository, 0, len(repos))
for _, repo := range repos {
result = append(result, provisioning.ExternalRepository{
Name: repo.Name,
Owner: repo.Owner,
URL: repo.URL,
})
}
return result, nil
}
var (
_ connection.Connection = (*Connection)(nil)
)
@@ -432,3 +432,120 @@ func TestConnection_Validate(t *testing.T) {
})
}
}
func TestConnection_ListRepositories(t *testing.T) {
t.Run("should list repositories successfully", func(t *testing.T) {
c := &provisioning.Connection{
ObjectMeta: metav1.ObjectMeta{Name: "test-connection"},
Spec: provisioning.ConnectionSpec{
Type: provisioning.GithubConnectionType,
GitHub: &provisioning.GitHubConnectionConfig{
AppID: "123",
InstallationID: "456",
},
},
Secure: provisioning.ConnectionSecure{
Token: common.InlineSecureValue{
Create: common.NewSecretValue("test-token"),
},
},
}
mockFactory := NewMockGithubFactory(t)
mockClient := NewMockClient(t)
mockFactory.EXPECT().New(mock.Anything, common.RawSecureValue("test-token")).Return(mockClient)
mockClient.EXPECT().ListInstallationRepositories(mock.Anything, "456").Return([]Repository{
{Name: "repo1", Owner: "owner1", URL: "https://github.com/owner1/repo1"},
{Name: "repo2", Owner: "owner2", URL: "https://github.com/owner2/repo2"},
}, nil)
conn := NewConnection(c, mockFactory)
repos, err := conn.ListRepositories(context.Background())
require.NoError(t, err)
require.Len(t, repos, 2)
assert.Equal(t, "repo1", repos[0].Name)
assert.Equal(t, "owner1", repos[0].Owner)
assert.Equal(t, "https://github.com/owner1/repo1", repos[0].URL)
assert.Equal(t, "repo2", repos[1].Name)
assert.Equal(t, "owner2", repos[1].Owner)
assert.Equal(t, "https://github.com/owner2/repo2", repos[1].URL)
})
t.Run("should return error when GitHub config is nil", func(t *testing.T) {
c := &provisioning.Connection{
ObjectMeta: metav1.ObjectMeta{Name: "test-connection"},
Spec: provisioning.ConnectionSpec{
Type: provisioning.GitlabConnectionType,
},
}
mockFactory := NewMockGithubFactory(t)
conn := NewConnection(c, mockFactory)
_, err := conn.ListRepositories(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "github configuration is required")
})
t.Run("should return error when listing repositories fails", func(t *testing.T) {
c := &provisioning.Connection{
ObjectMeta: metav1.ObjectMeta{Name: "test-connection"},
Spec: provisioning.ConnectionSpec{
Type: provisioning.GithubConnectionType,
GitHub: &provisioning.GitHubConnectionConfig{
AppID: "123",
InstallationID: "456",
},
},
Secure: provisioning.ConnectionSecure{
Token: common.InlineSecureValue{
Create: common.NewSecretValue("test-token"),
},
},
}
mockFactory := NewMockGithubFactory(t)
mockClient := NewMockClient(t)
mockFactory.EXPECT().New(mock.Anything, common.RawSecureValue("test-token")).Return(mockClient)
mockClient.EXPECT().ListInstallationRepositories(mock.Anything, "456").Return(nil, assert.AnError)
conn := NewConnection(c, mockFactory)
_, err := conn.ListRepositories(context.Background())
require.Error(t, err)
assert.Contains(t, err.Error(), "list installation repositories")
})
t.Run("should return empty list when no repositories", func(t *testing.T) {
c := &provisioning.Connection{
ObjectMeta: metav1.ObjectMeta{Name: "test-connection"},
Spec: provisioning.ConnectionSpec{
Type: provisioning.GithubConnectionType,
GitHub: &provisioning.GitHubConnectionConfig{
AppID: "123",
InstallationID: "456",
},
},
Secure: provisioning.ConnectionSecure{
Token: common.InlineSecureValue{
Create: common.NewSecretValue("test-token"),
},
},
}
mockFactory := NewMockGithubFactory(t)
mockClient := NewMockClient(t)
mockFactory.EXPECT().New(mock.Anything, common.RawSecureValue("test-token")).Return(mockClient)
mockClient.EXPECT().ListInstallationRepositories(mock.Anything, "456").Return([]Repository{}, nil)
conn := NewConnection(c, mockFactory)
repos, err := conn.ListRepositories(context.Background())
require.NoError(t, err)
require.Len(t, repos, 0)
})
}
@@ -3,6 +3,7 @@ package provisioning
import (
"context"
"net/http"
"time"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
@@ -12,10 +13,14 @@ import (
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
)
type connectionRepositoriesConnector struct{}
type connectionRepositoriesConnector struct {
getter ConnectionGetter
}
func NewConnectionRepositoriesConnector() *connectionRepositoriesConnector {
return &connectionRepositoriesConnector{}
func NewConnectionRepositoriesConnector(getter ConnectionGetter) *connectionRepositoriesConnector {
return &connectionRepositoriesConnector{
getter: getter,
}
}
func (*connectionRepositoriesConnector) New() runtime.Object {
@@ -43,23 +48,34 @@ func (*connectionRepositoriesConnector) NewConnectOptions() (runtime.Object, boo
func (c *connectionRepositoriesConnector) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
logger := logging.FromContext(ctx).With("logger", "connection-repositories-connector", "connection_name", name)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return WithTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
responder.Error(apierrors.NewMethodNotSupported(provisioning.ConnectionResourceInfo.GroupResource(), r.Method))
return
}
logger.Debug("repositories endpoint called but not yet implemented")
logger.Debug("listing repositories from connection")
// TODO: Implement repository listing from external git provider
// This will require:
// 1. Get the Connection object using logging.Context(r.Context(), logger)
// 2. Use the connection credentials to authenticate with the git provider
// 3. List repositories from the provider (GitHub, GitLab, Bitbucket)
// 4. Return ExternalRepositoryList with Name, Owner, and URL for each repository
conn, err := c.getter.GetConnection(r.Context(), name)
if err != nil {
logger.Error("failed to get connection", "error", err)
responder.Error(err)
return
}
responder.Error(apierrors.NewMethodNotSupported(provisioning.ConnectionResourceInfo.GroupResource(), "repositories endpoint not yet implemented"))
}), nil
repos, err := conn.ListRepositories(r.Context())
if err != nil {
logger.Error("failed to list repositories", "error", err)
responder.Error(apierrors.NewInternalError(err))
return
}
result := &provisioning.ExternalRepositoryList{
Items: repos,
}
responder.Object(http.StatusOK, result)
}), 30*time.Second), nil
}
var (
@@ -2,6 +2,7 @@ package provisioning
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
@@ -11,10 +12,40 @@ import (
"k8s.io/apimachinery/pkg/runtime"
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/apps/provisioning/pkg/connection"
)
// mockConnectionGetter implements ConnectionGetter for testing
type mockConnectionGetter struct {
conn connection.Connection
err error
}
func (m *mockConnectionGetter) GetConnection(ctx context.Context, name string) (connection.Connection, error) {
return m.conn, m.err
}
// mockConnection implements connection.Connection for testing
type mockConnection struct {
repos []provisioning.ExternalRepository
err error
}
func (m *mockConnection) Validate(ctx context.Context) error {
return nil
}
func (m *mockConnection) Mutate(ctx context.Context) error {
return nil
}
func (m *mockConnection) ListRepositories(ctx context.Context) ([]provisioning.ExternalRepository, error) {
return m.repos, m.err
}
func TestConnectionRepositoriesConnector(t *testing.T) {
connector := NewConnectionRepositoriesConnector()
mockGetter := &mockConnectionGetter{}
connector := NewConnectionRepositoriesConnector(mockGetter)
t.Run("New returns ExternalRepositoryList", func(t *testing.T) {
obj := connector.New()
@@ -61,23 +92,75 @@ func TestConnectionRepositoriesConnector(t *testing.T) {
require.True(t, apierrors.IsMethodNotSupported(responder.err))
})
t.Run("Connect returns handler that returns not implemented for GET", func(t *testing.T) {
t.Run("Connect returns handler that returns error when connection not found", func(t *testing.T) {
ctx := context.Background()
responder := &mockResponder{}
mockGetter.conn = nil
mockGetter.err = apierrors.NewNotFound(provisioning.ConnectionResourceInfo.GroupResource(), "test-connection")
handler, err := connector.Connect(ctx, "test-connection", nil, responder)
require.NoError(t, err)
require.NotNil(t, handler)
// Test GET method (should return not implemented)
req := httptest.NewRequest(http.MethodGet, "/", nil)
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
require.True(t, responder.called)
require.NotNil(t, responder.err)
require.True(t, apierrors.IsMethodNotSupported(responder.err))
require.Contains(t, responder.err.Error(), "not yet implemented")
require.True(t, apierrors.IsNotFound(responder.err))
})
t.Run("Connect returns handler that lists repositories successfully", func(t *testing.T) {
ctx := context.Background()
responder := &mockResponder{}
expectedRepos := []provisioning.ExternalRepository{
{Name: "repo1", Owner: "owner1", URL: "https://github.com/owner1/repo1"},
{Name: "repo2", Owner: "owner2", URL: "https://github.com/owner2/repo2"},
}
mockGetter.conn = &mockConnection{repos: expectedRepos}
mockGetter.err = nil
handler, err := connector.Connect(ctx, "test-connection", nil, responder)
require.NoError(t, err)
require.NotNil(t, handler)
req := httptest.NewRequest(http.MethodGet, "/", nil)
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
require.True(t, responder.called)
require.Nil(t, responder.err)
require.Equal(t, http.StatusOK, responder.code)
require.NotNil(t, responder.obj)
repoList, ok := responder.obj.(*provisioning.ExternalRepositoryList)
require.True(t, ok)
require.Len(t, repoList.Items, 2)
require.Equal(t, expectedRepos, repoList.Items)
})
t.Run("Connect returns handler that returns error when listing repositories fails", func(t *testing.T) {
ctx := context.Background()
responder := &mockResponder{}
mockGetter.conn = &mockConnection{err: errors.New("github API error")}
mockGetter.err = nil
handler, err := connector.Connect(ctx, "test-connection", nil, responder)
require.NoError(t, err)
require.NotNil(t, handler)
req := httptest.NewRequest(http.MethodGet, "/", nil)
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
require.True(t, responder.called)
require.NotNil(t, responder.err)
require.True(t, apierrors.IsInternalError(responder.err))
})
}
+11 -1
View File
@@ -98,6 +98,7 @@ type APIBuilder struct {
tracer tracing.Tracer
store grafanarest.Storage
connectionStore grafanarest.Storage
parsers resources.ParserFactory
repositoryResources resources.RepositoryResourcesFactory
clients resources.ClientFactory
@@ -636,6 +637,7 @@ func (b *APIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupI
return fmt.Errorf("failed to create connection storage: %w", err)
}
connectionStatusStorage := grafanaregistry.NewRegistryStatusStore(opts.Scheme, connectionsStore)
b.connectionStore = connectionsStore
storage[provisioning.JobResourceInfo.StoragePath()] = jobStore
storage[provisioning.RepositoryResourceInfo.StoragePath()] = repositoryStorage
@@ -643,7 +645,7 @@ func (b *APIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupI
storage[provisioning.ConnectionResourceInfo.StoragePath()] = connectionsStore
storage[provisioning.ConnectionResourceInfo.StoragePath("status")] = connectionStatusStorage
storage[provisioning.ConnectionResourceInfo.StoragePath("repositories")] = NewConnectionRepositoriesConnector()
storage[provisioning.ConnectionResourceInfo.StoragePath("repositories")] = NewConnectionRepositoriesConnector(b)
// TODO: Add some logic so that the connectors can registered themselves and we don't have logic all over the place
storage[provisioning.RepositoryResourceInfo.StoragePath("test")] = NewTestConnector(b, repository.NewRepositoryTesterWithExistingChecker(repository.NewSimpleRepositoryTester(b.repoValidator), b.VerifyAgainstExistingRepositories))
@@ -1418,6 +1420,14 @@ func (b *APIBuilder) GetRepository(ctx context.Context, name string) (repository
return b.asRepository(ctx, obj, nil)
}
func (b *APIBuilder) GetConnection(ctx context.Context, name string) (connection.Connection, error) {
obj, err := b.connectionStore.Get(ctx, name, &metav1.GetOptions{})
if err != nil {
return nil, err
}
return b.asConnection(ctx, obj, nil)
}
func (b *APIBuilder) GetRepoFactory() repository.Factory {
return b.repoFactory
}
+6
View File
@@ -3,6 +3,7 @@ package provisioning
import (
"context"
"github.com/grafana/grafana/apps/provisioning/pkg/connection"
client "github.com/grafana/grafana/apps/provisioning/pkg/generated/clientset/versioned/typed/provisioning/v0alpha1"
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
)
@@ -15,6 +16,11 @@ type RepoGetter interface {
GetHealthyRepository(ctx context.Context, name string) (repository.Repository, error)
}
type ConnectionGetter interface {
// This gets a connection with the provided name in the namespace from ctx
GetConnection(ctx context.Context, name string) (connection.Connection, error)
}
type ClientGetter interface {
GetClient() client.ProvisioningV0alpha1Interface
}