Provisioning: introduce concept of provisioning extras (#104981)

* Spike: Extras

* Attempt to wire it up

* Hack

* Fix issue with jobs

* Wire more things up

* Fix more wiring stuff

* Remove webhook secret key from main registration

* Move secret encryption also outside register

* Add TODOs in code

* Add more explanations

* Move connectors to different package

* Move pull request job into webhooks

* Separate registration

* Remove duplicate files

* Fix missing function

* Extract webhook repository logic out of the core github repository

* Use status patcher in webhook connector

* Fix change in go mod

* Change hooks signature

* Remove TODOs

* Remove Webhook methos from go-git

* Remove leftover

* Fix mistake in OpenAPI spec

* Fix some tests

* Fix some issues

* Fix linting
This commit is contained in:
Roberto Jiménez Sánchez
2025-05-13 09:50:43 +02:00
committed by GitHub
parent 98df41235d
commit 047499a363
52 changed files with 2696 additions and 2299 deletions
@@ -310,7 +310,7 @@ func (rc *RepositoryController) shouldResync(obj *provisioning.Repository) bool
return obj.Spec.Sync.Enabled && syncAge >= (syncInterval-tolerance) && !pendingForTooLong && !isRunning
}
func (rc *RepositoryController) runHooks(ctx context.Context, repo repository.Repository, obj *provisioning.Repository) (*provisioning.WebhookStatus, error) {
func (rc *RepositoryController) runHooks(ctx context.Context, repo repository.Repository, obj *provisioning.Repository) ([]map[string]interface{}, error) {
logger := logging.FromContext(ctx)
hooks, _ := repo.(repository.Hooks)
if hooks == nil || obj.Generation == obj.Status.ObservedGeneration {
@@ -319,20 +319,20 @@ func (rc *RepositoryController) runHooks(ctx context.Context, repo repository.Re
if obj.Status.ObservedGeneration < 1 {
logger.Info("handle repository create")
webhookStatus, err := hooks.OnCreate(ctx)
patchOperations, err := hooks.OnCreate(ctx)
if err != nil {
return nil, fmt.Errorf("error running OnCreate: %w", err)
}
return webhookStatus, nil
return patchOperations, nil
}
logger.Info("handle repository spec update", "Generation", obj.Generation, "ObservedGeneration", obj.Status.ObservedGeneration)
webhookStatus, err := hooks.OnUpdate(ctx)
patchOperations, err := hooks.OnUpdate(ctx)
if err != nil {
return nil, fmt.Errorf("error running OnUpdate: %w", err)
}
return webhookStatus, nil
return patchOperations, nil
}
func (rc *RepositoryController) determineSyncStrategy(ctx context.Context, obj *provisioning.Repository, shouldResync bool, healthStatus provisioning.HealthStatus) *provisioning.SyncJobOptions {
@@ -498,16 +498,12 @@ func (rc *RepositoryController) process(item *queueItem) error {
}
// Run hooks
webhookStatus, err := rc.runHooks(ctx, repo, obj)
hookOps, err := rc.runHooks(ctx, repo, obj)
switch {
case err != nil:
return err
case webhookStatus != nil:
patchOperations = append(patchOperations, map[string]interface{}{
"op": "replace",
"path": "/status/webhook",
"value": webhookStatus,
})
case len(hookOps) > 0:
patchOperations = append(patchOperations, hookOps...)
}
// determine the sync strategy and sync status to apply
@@ -21,7 +21,7 @@ func NewRepositoryStatusPatcher(client client.ProvisioningV0alpha1Interface) *Re
}
}
func (r *RepositoryStatusPatcher) Patch(ctx context.Context, repo *provisioning.Repository, patchOperations []map[string]interface{}) error {
func (r *RepositoryStatusPatcher) Patch(ctx context.Context, repo *provisioning.Repository, patchOperations ...map[string]interface{}) error {
patch, err := json.Marshal(patchOperations)
if err != nil {
return fmt.Errorf("unable to marshal patch data: %w", err)
@@ -104,7 +104,7 @@ func TestRepositoryStatusPatcher_Patch(t *testing.T) {
}
patcher := NewRepositoryStatusPatcher(&client)
err := patcher.Patch(context.Background(), tt.repo, tt.patchOperations)
err := patcher.Patch(context.Background(), tt.repo, tt.patchOperations...)
if tt.expectedError != "" {
require.EqualError(t, err, tt.expectedError)
+23
View File
@@ -0,0 +1,23 @@
package provisioning
import (
"context"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/kube-openapi/pkg/spec3"
)
type Extra interface {
Authorize(ctx context.Context, a authorizer.Attributes) (decision authorizer.Decision, reason string, err error)
Mutate(ctx context.Context, r *provisioning.Repository) error
UpdateStorage(storage map[string]rest.Storage) error
PostProcessOpenAPI(oas *spec3.OpenAPI) error
GetJobWorkers() []jobs.Worker
AsRepository(ctx context.Context, r *provisioning.Repository) (repository.Repository, error)
}
type ExtraBuilder func(b *APIBuilder) Extra
+1 -1
View File
@@ -90,7 +90,7 @@ func (c *filesConnector) Connect(ctx context.Context, name string, opts runtime.
folders := resources.NewFolderManager(readWriter, folderClient, resources.NewEmptyFolderTree())
dualReadWriter := resources.NewDualReadWriter(readWriter, parser, folders, c.access)
return withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return WithTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
opts := resources.DualWriteOptions{
Ref: query.Get("ref"),
+1 -1
View File
@@ -60,7 +60,7 @@ func (h *historySubresource) Connect(ctx context.Context, name string, opts runt
return nil, err
}
return withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return WithTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
versioned, ok := repo.(repository.Versioned)
if !ok {
responder.Error(apierrors.NewBadRequest("this repository does not support history"))
+18 -2
View File
@@ -55,14 +55,14 @@ func (c *jobsConnector) Connect(
}
cfg := repo.Config()
return withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return WithTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx = r.Context()
prefix := fmt.Sprintf("/%s/jobs/", name)
idx := strings.Index(r.URL.Path, prefix)
if r.Method == http.MethodGet {
if idx > 0 {
jobUID := r.URL.Path[idx+len(prefix):]
if !validBlobID(jobUID) {
if !ValidUUID(jobUID) {
responder.Error(apierrors.NewBadRequest(fmt.Sprintf("invalid job uid: %s", jobUID)))
return
}
@@ -108,3 +108,19 @@ var (
_ rest.Storage = (*jobsConnector)(nil)
_ rest.StorageMetadata = (*jobsConnector)(nil)
)
// ValidUUID ensures the ID is valid for a blob.
// The ID is always a UUID. As such, this checks for something that can resemble a UUID.
// This does not check for the ID to be an actual UUID, as the blob store may change their ID format, which we do not wish to stand in the way of.
func ValidUUID(id string) bool {
for _, c := range id {
// [a-zA-Z0-9\-] are valid characters.
az := c >= 'a' && c <= 'z'
AZ := c >= 'A' && c <= 'Z'
digit := c >= '0' && c <= '9'
if !az && !AZ && !digit && c != '-' {
return false
}
}
return true
}
@@ -22,17 +22,24 @@ func (_m *MockRepositoryPatchFn) EXPECT() *MockRepositoryPatchFn_Expecter {
return &MockRepositoryPatchFn_Expecter{mock: &_m.Mock}
}
// Execute provides a mock function with given fields: ctx, repo, ops
func (_m *MockRepositoryPatchFn) Execute(ctx context.Context, repo *v0alpha1.Repository, ops []map[string]interface{}) error {
ret := _m.Called(ctx, repo, ops)
// Execute provides a mock function with given fields: ctx, repo, patchOperations
func (_m *MockRepositoryPatchFn) Execute(ctx context.Context, repo *v0alpha1.Repository, patchOperations ...map[string]interface{}) error {
_va := make([]interface{}, len(patchOperations))
for _i := range patchOperations {
_va[_i] = patchOperations[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, repo)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for Execute")
}
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, *v0alpha1.Repository, []map[string]interface{}) error); ok {
r0 = rf(ctx, repo, ops)
if rf, ok := ret.Get(0).(func(context.Context, *v0alpha1.Repository, ...map[string]interface{}) error); ok {
r0 = rf(ctx, repo, patchOperations...)
} else {
r0 = ret.Error(0)
}
@@ -48,14 +55,21 @@ type MockRepositoryPatchFn_Execute_Call struct {
// Execute is a helper method to define mock.On call
// - ctx context.Context
// - repo *v0alpha1.Repository
// - ops []map[string]interface{}
func (_e *MockRepositoryPatchFn_Expecter) Execute(ctx interface{}, repo interface{}, ops interface{}) *MockRepositoryPatchFn_Execute_Call {
return &MockRepositoryPatchFn_Execute_Call{Call: _e.mock.On("Execute", ctx, repo, ops)}
// - patchOperations ...map[string]interface{}
func (_e *MockRepositoryPatchFn_Expecter) Execute(ctx interface{}, repo interface{}, patchOperations ...interface{}) *MockRepositoryPatchFn_Execute_Call {
return &MockRepositoryPatchFn_Execute_Call{Call: _e.mock.On("Execute",
append([]interface{}{ctx, repo}, patchOperations...)...)}
}
func (_c *MockRepositoryPatchFn_Execute_Call) Run(run func(ctx context.Context, repo *v0alpha1.Repository, ops []map[string]interface{})) *MockRepositoryPatchFn_Execute_Call {
func (_c *MockRepositoryPatchFn_Execute_Call) Run(run func(ctx context.Context, repo *v0alpha1.Repository, patchOperations ...map[string]interface{})) *MockRepositoryPatchFn_Execute_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*v0alpha1.Repository), args[2].([]map[string]interface{}))
variadicArgs := make([]map[string]interface{}, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(map[string]interface{})
}
}
run(args[0].(context.Context), args[1].(*v0alpha1.Repository), variadicArgs...)
})
return _c
}
@@ -65,7 +79,7 @@ func (_c *MockRepositoryPatchFn_Execute_Call) Return(_a0 error) *MockRepositoryP
return _c
}
func (_c *MockRepositoryPatchFn_Execute_Call) RunAndReturn(run func(context.Context, *v0alpha1.Repository, []map[string]interface{}) error) *MockRepositoryPatchFn_Execute_Call {
func (_c *MockRepositoryPatchFn_Execute_Call) RunAndReturn(run func(context.Context, *v0alpha1.Repository, ...map[string]interface{}) error) *MockRepositoryPatchFn_Execute_Call {
_c.Call.Return(run)
return _c
}
@@ -13,7 +13,7 @@ import (
)
//go:generate mockery --name RepositoryPatchFn --structname MockRepositoryPatchFn --inpackage --filename repository_patch_fn_mock.go --with-expecter
type RepositoryPatchFn func(ctx context.Context, repo *provisioning.Repository, ops []map[string]interface{}) error
type RepositoryPatchFn func(ctx context.Context, repo *provisioning.Repository, patchOperations ...map[string]interface{}) error
// SyncWorker synchronizes the external repo with grafana database
// this function updates the status for both the job and the referenced repository
@@ -82,7 +82,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
}
progress.SetMessage(ctx, "update sync status at start")
if err := r.patchStatus(ctx, cfg, patchOperations); err != nil {
if err := r.patchStatus(ctx, cfg, patchOperations...); err != nil {
return fmt.Errorf("update repo with job status at start: %w", err)
}
@@ -136,7 +136,7 @@ func (r *SyncWorker) Process(ctx context.Context, repo repository.Repository, jo
}
// Only patch the specific fields we want to update, not the entire status
if err := r.patchStatus(ctx, cfg, patchOperations); err != nil {
if err := r.patchStatus(ctx, cfg, patchOperations...); err != nil {
return fmt.Errorf("update repo with job final status: %w", err)
}
@@ -112,16 +112,12 @@ func TestSyncWorker_Process(t *testing.T) {
rw.MockRepository.On("Config").Return(repoConfig)
pr.On("SetMessage", mock.Anything, "update sync status at start").Return()
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool {
if len(patch) != 1 {
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch map[string]interface{}) bool {
if patch["op"] != "replace" || patch["path"] != "/status/sync" {
return false
}
if patch[0]["op"] != "replace" || patch[0]["path"] != "/status/sync" {
return false
}
if patch[0]["value"].(provisioning.SyncStatus).LastRef != "existing-ref" || patch[0]["value"].(provisioning.SyncStatus).JobID != "test-job" {
if patch["value"].(provisioning.SyncStatus).LastRef != "existing-ref" || patch["value"].(provisioning.SyncStatus).JobID != "test-job" {
return false
}
@@ -238,15 +234,12 @@ func TestSyncWorker_Process(t *testing.T) {
pr.On("SetMessage", mock.Anything, "update status and stats").Return()
// Final patch should include new ref
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool {
if len(patch) != 1 {
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch map[string]interface{}) bool {
if patch["op"] != "replace" || patch["path"] != "/status/sync" {
return false
}
syncStatus := patch[0]["value"].(provisioning.SyncStatus)
return patch[0]["op"] == "replace" &&
patch[0]["path"] == "/status/sync" &&
syncStatus.LastRef == "new-ref" &&
syncStatus.State == provisioning.JobStateSuccess
syncStatus := patch["value"].(provisioning.SyncStatus)
return syncStatus.LastRef == "new-ref" && syncStatus.State == provisioning.JobStateSuccess
})).Return(nil)
},
expectedError: "",
@@ -294,13 +287,10 @@ func TestSyncWorker_Process(t *testing.T) {
pr.On("SetMessage", mock.Anything, "update status and stats").Return()
// Final patch should preserve existing ref on failure
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch []map[string]interface{}) bool {
if len(patch) != 1 {
return false
}
syncStatus := patch[0]["value"].(provisioning.SyncStatus)
return patch[0]["op"] == "replace" &&
patch[0]["path"] == "/status/sync" &&
rpf.On("Execute", mock.Anything, repoConfig, mock.MatchedBy(func(patch map[string]interface{}) bool {
syncStatus := patch["value"].(provisioning.SyncStatus)
return patch["op"] == "replace" &&
patch["path"] == "/status/sync" &&
syncStatus.LastRef == "existing-ref" && // LastRef should not change on failure
syncStatus.State == provisioning.JobStateError
})).Return(nil)
@@ -350,8 +340,8 @@ func TestSyncWorker_Process(t *testing.T) {
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
// Verify only sync status is patched
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool {
return len(patch) == 1 && patch[0]["path"] == "/status/sync"
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch map[string]interface{}) bool {
return patch["path"] == "/status/sync"
})).Return(nil)
// Simple mocks for other calls
@@ -394,19 +384,14 @@ func TestSyncWorker_Process(t *testing.T) {
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
// Verify both sync status and stats are patched
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool {
if len(patch) != 2 {
return false
}
if patch[0]["path"] != "/status/sync" {
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch map[string]interface{}) bool {
return patch["path"] == "/status/sync"
}), mock.MatchedBy(func(patch map[string]interface{}) bool {
if patch["path"] != "/status/stats" {
return false
}
if patch[1]["path"] != "/status/stats" {
return false
}
value := patch[1]["value"].([]provisioning.ResourceCount)
value := patch["value"].([]provisioning.ResourceCount)
if len(value) != 1 {
return false
}
@@ -466,8 +451,8 @@ func TestSyncWorker_Process(t *testing.T) {
rrf.On("Client", mock.Anything, mock.Anything).Return(mockRepoResources, nil)
// Verify only sync status is patched (multiple stats should be ignored)
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch []map[string]interface{}) bool {
return len(patch) == 1 && patch[0]["path"] == "/status/sync"
rpf.On("Execute", mock.Anything, mock.Anything, mock.MatchedBy(func(patch map[string]interface{}) bool {
return patch["path"] == "/status/sync"
})).Return(nil)
// Simple mocks for other calls
+1 -1
View File
@@ -48,7 +48,7 @@ func (s *listConnector) Connect(ctx context.Context, name string, opts runtime.O
return nil, fmt.Errorf("missing namespace")
}
return withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return WithTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// TODO: Add pagination to resource lister
rsp, err := s.lister.List(ctx, ns, name)
if err != nil {
+90 -120
View File
@@ -43,7 +43,6 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/export"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/migrate"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/pullrequest"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs/sync"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository/github"
@@ -55,7 +54,6 @@ import (
"github.com/grafana/grafana/pkg/services/apiserver"
"github.com/grafana/grafana/pkg/services/apiserver/builder"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/rendering"
grafanasecrets "github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/legacysql/dualwrite"
@@ -74,14 +72,10 @@ var (
)
type APIBuilder struct {
urlProvider func(namespace string) string
webhookSecretKey string
isPublic bool
features featuremgmt.FeatureToggles
features featuremgmt.FeatureToggles
getter rest.Getter
localFileResolver *repository.LocalFolderResolver
render rendering.Service
parsers resources.ParserFactory
repositoryResources resources.RepositoryResourcesFactory
clients resources.ClientFactory
@@ -101,6 +95,9 @@ type APIBuilder struct {
secrets secrets.Service
client client.ProvisioningV0alpha1Interface
access authlib.AccessChecker
statusPatcher *controller.RepositoryStatusPatcher
// Extras provides additional functionality to the API.
extras []Extra
}
// NewAPIBuilder creates an API builder.
@@ -108,10 +105,7 @@ type APIBuilder struct {
// This means there are no hidden dependencies, and no use of e.g. *settings.Cfg.
func NewAPIBuilder(
local *repository.LocalFolderResolver,
urlProvider func(namespace string) string,
webhookSecretKey string,
features featuremgmt.FeatureToggles,
render rendering.Service,
unified resource.ResourceClient,
clonedir string, // where repo clones are managed
configProvider apiserver.RestConfigProvider,
@@ -120,25 +114,19 @@ func NewAPIBuilder(
storageStatus dualwrite.Service,
secrets secrets.Service,
access authlib.AccessChecker,
extraBuilders []ExtraBuilder,
) *APIBuilder {
// HACK: Assume is only public if it is HTTPS
isPublic := strings.HasPrefix(urlProvider(""), "https://")
clients := resources.NewClientFactory(configProvider)
parsers := resources.NewParserFactory(clients)
resourceLister := resources.NewResourceLister(unified, unified, legacyMigrator, storageStatus)
return &APIBuilder{
urlProvider: urlProvider,
b := &APIBuilder{
localFileResolver: local,
webhookSecretKey: webhookSecretKey,
isPublic: isPublic,
features: features,
ghFactory: ghFactory,
clients: clients,
parsers: parsers,
repositoryResources: resources.NewRepositoryResourcesFactory(parsers, clients, resourceLister),
render: render,
clonedir: clonedir,
resourceLister: resourceLister,
legacyMigrator: legacyMigrator,
@@ -148,6 +136,12 @@ func NewAPIBuilder(
access: access,
jobHistory: jobs.NewJobHistoryCache(),
}
for _, builder := range extraBuilders {
b.extras = append(b.extras, builder(b))
}
return b
}
// RegisterAPIService returns an API builder, from [NewAPIBuilder]. It is called by Wire.
@@ -158,7 +152,6 @@ func RegisterAPIService(
features featuremgmt.FeatureToggles,
apiregistration builder.APIRegistrar,
reg prometheus.Registerer,
render rendering.Service,
client resource.ResourceClient, // implements resource.RepositoryClient
configProvider apiserver.RestConfigProvider,
ghFactory *github.Factory,
@@ -168,6 +161,7 @@ func RegisterAPIService(
usageStatsService usagestats.Service,
// FIXME: use multi-tenant service when one exists. In this state, we can't make this a multi-tenant service!
secretsSvc grafanasecrets.Service,
extraBuilders []ExtraBuilder,
) (*APIBuilder, error) {
if !features.IsEnabledGlobally(featuremgmt.FlagProvisioning) {
return nil, nil
@@ -177,16 +171,13 @@ func RegisterAPIService(
PermittedPrefixes: cfg.PermittedProvisioningPaths,
HomePath: safepath.Clean(cfg.HomePath),
}
urlProvider := func(namespace string) string {
return cfg.AppURL
}
builder := NewAPIBuilder(folderResolver, urlProvider, cfg.SecretKey, features,
render, client,
builder := NewAPIBuilder(folderResolver, features,
client,
filepath.Join(cfg.DataPath, "clone"), // where repositories are cloned (temporarialy for now)
configProvider, ghFactory,
legacyMigrator, storageStatus,
secrets.NewSingleTenant(secretsSvc), access,
extraBuilders,
)
apiregistration.RegisterAPI(builder)
usageStatsService.RegisterMetricsFunc(builder.collectProvisioningStats)
@@ -219,6 +210,14 @@ func (b *APIBuilder) GetAuthorizer() authorizer.Authorizer {
return authorizer.DecisionDeny, "failed to find requester", err
}
// Check if any extra authorizer has a decision.
for _, extra := range b.extras {
decision, reason, err := extra.Authorize(ctx, a)
if decision != authorizer.DecisionNoOpinion {
return decision, reason, err
}
}
switch a.GetResource() {
case provisioning.RepositoryResourceInfo.GetName():
// TODO: Support more fine-grained permissions than the basic roles. Especially on Enterprise.
@@ -230,21 +229,10 @@ func (b *APIBuilder) GetAuthorizer() authorizer.Authorizer {
}
return authorizer.DecisionDeny, "admin role is required", nil
case "webhook":
// When the resource is a webhook, we'll deal with permissions manually by checking signatures or similar in the webhook handler.
// The user in this context is usually an anonymous user, but may also be an authenticated synthetic check by the Grafana instance's operator as well.
// For context on the anonymous user, check the authn/clients/provisioning.go file.
return authorizer.DecisionAllow, "", nil
case "files":
// Access to files is controlled by the AccessClient
return authorizer.DecisionAllow, "", nil
case "render":
// This is used to read a blob from unified storage, for GitHub PR comments.
// GH uses a proxy for all images, so we need to accept it, always.
return authorizer.DecisionAllow, "", nil
case "resources", "sync", "history":
// These are strictly read operations.
// Sync can also be somewhat destructive, but it's expected to be fine to import changes.
@@ -300,6 +288,14 @@ func (b *APIBuilder) GetClient() client.ProvisioningV0alpha1Interface {
return b.client
}
func (b *APIBuilder) GetJobQueue() jobs.Queue {
return b.jobs
}
func (b *APIBuilder) GetStatusPatcher() *controller.RepositoryStatusPatcher {
return b.statusPatcher
}
func (b *APIBuilder) InstallSchema(scheme *runtime.Scheme) error {
err := provisioning.AddToScheme(scheme)
if err != nil {
@@ -345,7 +341,6 @@ func (b *APIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupI
// TODO: Add some logic so that the connectors can registered themselves and we don't have logic all over the place
// TODO: Do not set private fields directly, use factory methods.
storage[provisioning.RepositoryResourceInfo.StoragePath("webhook")] = NewWebhookConnector(b, b, b.jobs, b.isPublic)
storage[provisioning.RepositoryResourceInfo.StoragePath("test")] = &testConnector{
getter: b,
}
@@ -362,9 +357,14 @@ func (b *APIBuilder) UpdateAPIGroupInfo(apiGroupInfo *genericapiserver.APIGroupI
jobs: b.jobs,
historic: b.jobHistory,
}
storage[provisioning.RepositoryResourceInfo.StoragePath("render")] = &renderConnector{
blob: b.unified,
// Add any extra storage
for _, extra := range b.extras {
if err := extra.UpdateStorage(storage); err != nil {
return fmt.Errorf("update storage for extra %T: %w", extra, err)
}
}
apiGroupInfo.VersionedResourcesStorageMap[provisioning.VERSION] = storage
return nil
}
@@ -411,10 +411,32 @@ func (b *APIBuilder) Mutate(ctx context.Context, a admission.Attributes, o admis
r.Spec.Workflows = []provisioning.Workflow{}
}
if err := b.encryptSecrets(ctx, r); err != nil {
if err := b.encryptGithubToken(ctx, r); err != nil {
return fmt.Errorf("failed to encrypt secrets: %w", err)
}
// Mutate the repository with any extra mutators
for _, extra := range b.extras {
if err := extra.Mutate(ctx, r); err != nil {
return fmt.Errorf("failed to mutate repository: %w", err)
}
}
return nil
}
// TODO: move this to a more appropriate place
func (b *APIBuilder) encryptGithubToken(ctx context.Context, repo *provisioning.Repository) error {
var err error
if repo.Spec.GitHub != nil &&
repo.Spec.GitHub.Token != "" {
repo.Spec.GitHub.EncryptedToken, err = b.secrets.Encrypt(ctx, []byte(repo.Spec.GitHub.Token))
if err != nil {
return err
}
repo.Spec.GitHub.Token = ""
}
return nil
}
@@ -538,13 +560,13 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
repository.WrapWithCloneAndPushIfPossible,
)
statusPatcher := controller.NewRepositoryStatusPatcher(b.GetClient())
b.statusPatcher = controller.NewRepositoryStatusPatcher(b.GetClient())
syncer := sync.NewSyncer(sync.Compare, sync.FullSync, sync.IncrementalSync)
syncWorker := sync.NewSyncWorker(
b.clients,
b.repositoryResources,
b.storageStatus,
statusPatcher.Patch,
b.statusPatcher.Patch,
syncer,
)
signerFactory := signature.NewSignerFactory(b.clients)
@@ -577,18 +599,20 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
b.storageStatus,
)
// Pull request worker
renderer := pullrequest.NewScreenshotRenderer(b.render, b.unified)
evaluator := pullrequest.NewEvaluator(renderer, b.parsers, b.urlProvider)
commenter := pullrequest.NewCommenter()
pullRequestWorker := pullrequest.NewPullRequestWorker(evaluator, commenter)
workers := []jobs.Worker{migrationWorker, syncWorker, exportWorker}
// Add any extra workers
for _, extra := range b.extras {
workers = append(workers, extra.GetJobWorkers()...)
}
driver, err := jobs.NewJobDriver(
time.Minute*20, // Max time for each job
time.Minute*22, // Cleanup any checked out jobs. FIXME: this is slow if things crash/fail!
time.Second*30, // Periodically look for new jobs
b.jobs, b, b.jobHistory,
exportWorker, syncWorker, migrationWorker, pullRequestWorker)
workers...,
)
if err != nil {
return err
}
@@ -614,6 +638,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
return nil
},
}
return postStartHooks, nil
}
@@ -650,11 +675,6 @@ func (b *APIBuilder) PostProcessOpenAPI(oas *spec3.OpenAPI) (*spec3.OpenAPI, err
}
}
sub = oas.Paths.Paths[repoprefix+"/webhook"]
if sub != nil && sub.Get != nil {
sub.Post.Description = "Currently only supports github webhooks"
}
ref := &spec3.Parameter{
ParameterProps: spec3.ParameterProps{
Name: "ref",
@@ -883,36 +903,11 @@ spec:
oas.Paths.Paths[repoprefix+"/jobs/{uid}"] = sub
}
delete(oas.Paths.Paths, repoprefix+"/render")
sub = oas.Paths.Paths[repoprefix+"/render/{path}"]
if sub != nil {
sub.Get.Description = "get a rendered preview image"
sub.Get.Responses = &spec3.Responses{
ResponsesProps: spec3.ResponsesProps{
StatusCodeResponses: map[int]*spec3.Response{
200: {
ResponseProps: spec3.ResponseProps{
Content: map[string]*spec3.MediaType{
"image/png": {},
},
Description: "OK",
},
},
},
},
// Run all extra post-processors.
for _, extra := range b.extras {
if err := extra.PostProcessOpenAPI(oas); err != nil {
return nil, fmt.Errorf("post-process OpenAPI for extra %T: %w", extra, err)
}
// Replace {path} with {guid} (it is a GUID, but all k8s sub-resources are called path)
for _, v := range sub.Parameters {
if v.Name == "path" {
v.Name = "guid"
v.Description = "Image GUID"
break
}
}
delete(oas.Paths.Paths, repoprefix+"/render/{path}")
oas.Paths.Paths[repoprefix+"/render/{guid}"] = sub
}
// Add any missing definitions
@@ -981,29 +976,6 @@ spec:
return oas, nil
}
// TODO: move this to a more appropriate place
func (b *APIBuilder) encryptSecrets(ctx context.Context, repo *provisioning.Repository) error {
var err error
if repo.Spec.GitHub != nil &&
repo.Spec.GitHub.Token != "" {
repo.Spec.GitHub.EncryptedToken, err = b.secrets.Encrypt(ctx, []byte(repo.Spec.GitHub.Token))
if err != nil {
return err
}
repo.Spec.GitHub.Token = ""
}
if repo.Status.Webhook != nil &&
repo.Status.Webhook.Secret != "" {
repo.Status.Webhook.EncryptedSecret, err = b.secrets.Encrypt(ctx, []byte(repo.Status.Webhook.Secret))
if err != nil {
return err
}
repo.Status.Webhook.Secret = ""
}
return nil
}
// FIXME: This logic does not belong in provisioning! (but required for now)
// When starting an empty instance, we shift so that we never reference legacy storage
// This should run somewhere else at startup by default (dual writer? dashboards?)
@@ -1131,28 +1103,26 @@ func (b *APIBuilder) asRepository(ctx context.Context, obj runtime.Object) (repo
}
func (b *APIBuilder) AsRepository(ctx context.Context, r *provisioning.Repository) (repository.Repository, error) {
// Try first with any extra
for _, extra := range b.extras {
r, err := extra.AsRepository(ctx, r)
if err != nil {
return nil, fmt.Errorf("convert repository for extra %T: %w", extra, err)
}
if r != nil {
return r, nil
}
}
switch r.Spec.Type {
case provisioning.LocalRepositoryType:
return repository.NewLocal(r, b.localFileResolver), nil
case provisioning.GitHubRepositoryType:
gvr := provisioning.RepositoryResourceInfo.GroupVersionResource()
var webhookURL string
if b.isPublic {
webhookURL = fmt.Sprintf(
"%sapis/%s/%s/namespaces/%s/%s/%s/webhook",
b.urlProvider(r.GetNamespace()),
gvr.Group,
gvr.Version,
r.GetNamespace(),
gvr.Resource,
r.GetName(),
)
}
cloneFn := func(ctx context.Context, opts repository.CloneOptions) (repository.ClonedRepository, error) {
return gogit.Clone(ctx, b.clonedir, r, opts, b.secrets)
}
return repository.NewGitHub(ctx, r, b.ghFactory, b.secrets, webhookURL, cloneFn)
return repository.NewGitHub(ctx, r, b.ghFactory, b.secrets, cloneFn)
default:
return nil, fmt.Errorf("unknown repository type (%s)", r.Spec.Type)
}
@@ -8,11 +8,8 @@ import (
"net/http"
"net/url"
"regexp"
"slices"
"strings"
"github.com/google/go-github/v70/github"
"github.com/google/uuid"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/validation/field"
@@ -24,14 +21,11 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/provisioning/secrets"
)
var subscribedEvents = []string{"push", "pull_request"}
// Make sure all public functions of this struct call the (*githubRepository).logger function, to ensure the GH repo details are included.
type githubRepository struct {
config *provisioning.Repository
gh pgh.Client // assumes github.com base URL
secrets secrets.Service
webhookURL string
config *provisioning.Repository
gh pgh.Client // assumes github.com base URL
secrets secrets.Service
owner string
repo string
@@ -39,24 +33,27 @@ type githubRepository struct {
cloneFn CloneFn
}
var (
_ Repository = (*githubRepository)(nil)
_ Hooks = (*githubRepository)(nil)
_ Versioned = (*githubRepository)(nil)
_ Writer = (*githubRepository)(nil)
_ Reader = (*githubRepository)(nil)
_ RepositoryWithURLs = (*githubRepository)(nil)
_ ClonableRepository = (*githubRepository)(nil)
)
// GithubRepository is an interface that combines all repository capabilities
// needed for GitHub repositories.
type GithubRepository interface {
Repository
Versioned
Writer
Reader
RepositoryWithURLs
ClonableRepository
Owner() string
Repo() string
Client() pgh.Client
}
func NewGitHub(
ctx context.Context,
config *provisioning.Repository,
factory *pgh.Factory,
secrets secrets.Service,
webhookURL string,
cloneFn CloneFn,
) (*githubRepository, error) {
) (GithubRepository, error) {
owner, repo, err := parseOwnerRepo(config.Spec.GitHub.URL)
if err != nil {
return nil, fmt.Errorf("parse owner and repo: %w", err)
@@ -72,13 +69,12 @@ func NewGitHub(
}
return &githubRepository{
config: config,
gh: factory.New(ctx, token), // TODO, baseURL from config
secrets: secrets,
webhookURL: webhookURL,
owner: owner,
repo: repo,
cloneFn: cloneFn,
config: config,
gh: factory.New(ctx, token), // TODO, baseURL from config
secrets: secrets,
owner: owner,
repo: repo,
cloneFn: cloneFn,
}, nil
}
@@ -86,6 +82,18 @@ func (r *githubRepository) Config() *provisioning.Repository {
return r.config
}
func (r *githubRepository) Owner() string {
return r.owner
}
func (r *githubRepository) Repo() string {
return r.repo
}
func (r *githubRepository) Client() pgh.Client {
return r.gh
}
// Validate implements provisioning.Repository.
func (r *githubRepository) Validate() (list field.ErrorList) {
gh := r.config.Spec.GitHub
@@ -512,130 +520,6 @@ func (r *githubRepository) ensureBranchExists(ctx context.Context, branchName st
return nil
}
// Webhook implements Repository.
func (r *githubRepository) Webhook(ctx context.Context, req *http.Request) (*provisioning.WebhookResponse, error) {
if r.config.Status.Webhook == nil {
return nil, fmt.Errorf("unexpected webhook request")
}
secret, err := r.secrets.Decrypt(ctx, r.config.Status.Webhook.EncryptedSecret)
if err != nil {
return nil, fmt.Errorf("failed to decrypt secret: %w", err)
}
payload, err := github.ValidatePayload(req, secret)
if err != nil {
return nil, apierrors.NewUnauthorized("invalid signature")
}
return r.parseWebhook(github.WebHookType(req), payload)
}
// This method does not include context because it does delegate any more requests
func (r *githubRepository) parseWebhook(messageType string, payload []byte) (*provisioning.WebhookResponse, error) {
event, err := github.ParseWebHook(messageType, payload)
if err != nil {
return nil, apierrors.NewBadRequest("invalid payload")
}
switch event := event.(type) {
case *github.PushEvent:
return r.parsePushEvent(event)
case *github.PullRequestEvent:
return r.parsePullRequestEvent(event)
case *github.PingEvent:
return &provisioning.WebhookResponse{
Code: http.StatusOK,
Message: "ping received",
}, nil
default:
return &provisioning.WebhookResponse{
Code: http.StatusNotImplemented,
Message: fmt.Sprintf("unsupported messageType: %s", messageType),
}, nil
}
}
func (r *githubRepository) parsePushEvent(event *github.PushEvent) (*provisioning.WebhookResponse, error) {
if event.GetRepo() == nil {
return nil, fmt.Errorf("missing repository in push event")
}
if event.GetRepo().GetFullName() != fmt.Sprintf("%s/%s", r.owner, r.repo) {
return nil, fmt.Errorf("repository mismatch")
}
// No need to sync if not enabled
if !r.config.Spec.Sync.Enabled {
return &provisioning.WebhookResponse{Code: http.StatusOK}, nil
}
// Skip silently if the event is not for the main/master branch
// as we cannot configure the webhook to only publish events for the main branch
if event.GetRef() != fmt.Sprintf("refs/heads/%s", r.config.Spec.GitHub.Branch) {
return &provisioning.WebhookResponse{Code: http.StatusOK}, nil
}
return &provisioning.WebhookResponse{
Code: http.StatusAccepted,
Job: &provisioning.JobSpec{
Repository: r.Config().GetName(),
Action: provisioning.JobActionPull,
Pull: &provisioning.SyncJobOptions{
Incremental: true,
},
},
}, nil
}
func (r *githubRepository) parsePullRequestEvent(event *github.PullRequestEvent) (*provisioning.WebhookResponse, error) {
if event.GetRepo() == nil {
return nil, fmt.Errorf("missing repository in pull request event")
}
cfg := r.config.Spec.GitHub
if cfg == nil {
return nil, fmt.Errorf("missing GitHub config")
}
if event.GetRepo().GetFullName() != fmt.Sprintf("%s/%s", r.owner, r.repo) {
return nil, fmt.Errorf("repository mismatch")
}
pr := event.GetPullRequest()
if pr == nil {
return nil, fmt.Errorf("expected PR in event")
}
if pr.GetBase().GetRef() != r.config.Spec.GitHub.Branch {
return &provisioning.WebhookResponse{
Code: http.StatusOK,
Message: fmt.Sprintf("ignoring pull request event as %s is not the configured branch", pr.GetBase().GetRef()),
}, nil
}
action := event.GetAction()
if action != "opened" && action != "reopened" && action != "synchronize" {
return &provisioning.WebhookResponse{
Code: http.StatusOK, // Nothing needed
Message: fmt.Sprintf("ignore pull request event: %s", action),
}, nil
}
// Queue an async job that will parse files
return &provisioning.WebhookResponse{
Code: http.StatusAccepted, // Nothing needed
Message: fmt.Sprintf("pull request: %s", action),
Job: &provisioning.JobSpec{
Repository: r.Config().GetName(),
Action: provisioning.JobActionPullRequest,
PullRequest: &provisioning.PullRequestJobOptions{
URL: pr.GetHTMLURL(),
PR: pr.GetNumber(),
Ref: pr.GetHead().GetRef(),
Hash: pr.GetHead().GetSHA(),
},
},
}, nil
}
func (r *githubRepository) LatestRef(ctx context.Context) (string, error) {
ctx, _ = r.logger(ctx, "")
branch, err := r.gh.GetBranch(ctx, r.owner, r.repo, r.Config().Spec.GitHub.Branch)
@@ -746,12 +630,6 @@ func (r *githubRepository) CompareFiles(ctx context.Context, base, ref string) (
return changes, nil
}
// CommentPullRequest adds a comment to a pull request.
func (r *githubRepository) CommentPullRequest(ctx context.Context, prNumber int, comment string) error {
ctx, _ = r.logger(ctx, "")
return r.gh.CreatePullRequestComment(ctx, r.owner, r.repo, prNumber, comment)
}
// ResourceURLs implements RepositoryWithURLs.
func (r *githubRepository) ResourceURLs(ctx context.Context, file *FileInfo) (*provisioning.ResourceURLs, error) {
cfg := r.config.Spec.GitHub
@@ -779,146 +657,6 @@ func (r *githubRepository) ResourceURLs(ctx context.Context, file *FileInfo) (*p
return urls, nil
}
func (r *githubRepository) createWebhook(ctx context.Context) (pgh.WebhookConfig, error) {
secret, err := uuid.NewRandom()
if err != nil {
return pgh.WebhookConfig{}, fmt.Errorf("could not generate secret: %w", err)
}
cfg := pgh.WebhookConfig{
URL: r.webhookURL,
Secret: secret.String(),
ContentType: "json",
Events: subscribedEvents,
Active: true,
}
hook, err := r.gh.CreateWebhook(ctx, r.owner, r.repo, cfg)
if err != nil {
return pgh.WebhookConfig{}, err
}
// HACK: GitHub does not return the secret, so we need to update it manually
hook.Secret = cfg.Secret
logging.FromContext(ctx).Info("webhook created", "url", cfg.URL, "id", hook.ID)
return hook, nil
}
// updateWebhook checks if the webhook needs to be updated and updates it if necessary.
// if the webhook does not exist, it will create it.
func (r *githubRepository) updateWebhook(ctx context.Context) (pgh.WebhookConfig, bool, error) {
if r.config.Status.Webhook == nil || r.config.Status.Webhook.ID == 0 {
hook, err := r.createWebhook(ctx)
if err != nil {
return pgh.WebhookConfig{}, false, err
}
return hook, true, nil
}
hook, err := r.gh.GetWebhook(ctx, r.owner, r.repo, r.config.Status.Webhook.ID)
switch {
case errors.Is(err, pgh.ErrResourceNotFound):
hook, err := r.createWebhook(ctx)
if err != nil {
return pgh.WebhookConfig{}, false, err
}
return hook, true, nil
case err != nil:
return pgh.WebhookConfig{}, false, fmt.Errorf("get webhook: %w", err)
}
hook.Secret = r.config.Status.Webhook.Secret // we always random gen this, so don't use it for mustUpdate below.
var mustUpdate bool
if hook.URL != r.webhookURL {
mustUpdate = true
hook.URL = r.webhookURL
}
if !slices.Equal(hook.Events, subscribedEvents) {
mustUpdate = true
hook.Events = subscribedEvents
}
if !mustUpdate {
return hook, false, nil
}
// Something has changed in the webhook. Let's rotate the secret as well, so as to ensure we end up with a 100% correct webhook.
secret, err := uuid.NewRandom()
if err != nil {
return pgh.WebhookConfig{}, false, fmt.Errorf("could not generate secret: %w", err)
}
hook.Secret = secret.String()
if err := r.gh.EditWebhook(ctx, r.owner, r.repo, hook); err != nil {
return pgh.WebhookConfig{}, false, fmt.Errorf("edit webhook: %w", err)
}
return hook, true, nil
}
func (r *githubRepository) deleteWebhook(ctx context.Context) error {
if r.config.Status.Webhook == nil {
return fmt.Errorf("webhook not found")
}
id := r.config.Status.Webhook.ID
if err := r.gh.DeleteWebhook(ctx, r.owner, r.repo, id); err != nil {
return fmt.Errorf("delete webhook: %w", err)
}
logging.FromContext(ctx).Info("webhook deleted", "url", r.config.Status.Webhook.URL, "id", id)
return nil
}
func (r *githubRepository) OnCreate(ctx context.Context) (*provisioning.WebhookStatus, error) {
if len(r.webhookURL) == 0 {
return nil, nil
}
ctx, _ = r.logger(ctx, "")
hook, err := r.createWebhook(ctx)
if err != nil {
return nil, err
}
return &provisioning.WebhookStatus{
ID: hook.ID,
URL: hook.URL,
Secret: hook.Secret,
SubscribedEvents: hook.Events,
}, nil
}
func (r *githubRepository) OnUpdate(ctx context.Context) (*provisioning.WebhookStatus, error) {
if len(r.webhookURL) == 0 {
return nil, nil
}
ctx, _ = r.logger(ctx, "")
hook, _, err := r.updateWebhook(ctx)
if err != nil {
return nil, err
}
return &provisioning.WebhookStatus{
ID: hook.ID,
URL: hook.URL,
Secret: hook.Secret,
SubscribedEvents: hook.Events,
}, nil
}
func (r *githubRepository) OnDelete(ctx context.Context) error {
if len(r.webhookURL) == 0 {
return nil
}
ctx, _ = r.logger(ctx, "")
return r.deleteWebhook(ctx)
}
func (r *githubRepository) Clone(ctx context.Context, opts CloneOptions) (ClonedRepository, error) {
return r.cloneFn(ctx, opts)
}
File diff suppressed because it is too large Load Diff
@@ -464,13 +464,3 @@ func (g *GoGitRepo) History(ctx context.Context, path string, ref string) ([]pro
func (g *GoGitRepo) Validate() field.ErrorList {
return nil
}
// Webhook implements repository.Repository.
func (g *GoGitRepo) Webhook(ctx context.Context, req *http.Request) (*provisioning.WebhookResponse, error) {
return nil, &apierrors.StatusError{
ErrStatus: metav1.Status{
Message: "history is not yet implemented",
Code: http.StatusNotImplemented,
},
}
}
@@ -8,7 +8,6 @@ import (
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"sort"
@@ -25,7 +24,6 @@ import (
"github.com/go-git/go-git/v5/plumbing/transport/server"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/go-git/go-git/v5/storage/memory"
@@ -206,31 +204,6 @@ func TestGoGitRepo_Validate(t *testing.T) {
require.Empty(t, errs, "Validate should return no errors")
}
func TestGoGitRepo_Webhook(t *testing.T) {
repo := &GoGitRepo{
config: &v0alpha1.Repository{
ObjectMeta: v1.ObjectMeta{
Name: "test",
Namespace: "default",
},
Spec: v0alpha1.RepositorySpec{
GitHub: &v0alpha1.GitHubRepositoryConfig{
Path: "grafana/",
},
},
},
}
// Test Webhook method
ctx := context.Background()
_, err := repo.Webhook(ctx, nil)
require.Error(t, err, "Webhook should return an error as it's not implemented")
var statusErr *apierrors.StatusError
require.True(t, errors.As(err, &statusErr), "Error should be a StatusError")
require.Equal(t, http.StatusNotImplemented, int(statusErr.ErrStatus.Code))
require.Contains(t, statusErr.ErrStatus.Message, "history is not yet implemented")
}
func TestGoGitRepo_Read(t *testing.T) {
// Setup test cases
tests := []struct {
@@ -22,68 +22,93 @@ import (
)
func TestLocalResolver(t *testing.T) {
resolver := &LocalFolderResolver{
PermittedPrefixes: []string{
"github",
},
HomePath: "./",
// Create a temporary directory structure
tempDir := t.TempDir()
// Create directory structure with multiple levels
dirs := []string{
"level1",
"level1/level2",
"level1/level2/level3",
"another/path",
}
fullpath, err := resolver.LocalPath("github/testdata")
require.NoError(t, err)
require.Equal(t, "github/testdata", fullpath)
for _, dir := range dirs {
dirPath := filepath.Join(tempDir, dir)
err := os.MkdirAll(dirPath, 0750)
require.NoError(t, err)
}
_, err = resolver.LocalPath("something")
require.Error(t, err)
// Create some files at different levels
files := map[string]string{
"root.txt": "root content",
"level1/file1.txt": "level 1 content",
"level1/level2/file2.txt": "level 2 content",
"level1/level2/level3/file3.txt": "level 3 content",
"another/path/file.txt": "another path content",
}
// Check valid errors
r := NewLocal(&provisioning.Repository{
for path, content := range files {
filePath := filepath.Join(tempDir, path)
err := os.WriteFile(filePath, []byte(content), 0644)
require.NoError(t, err)
}
// Create resolver with the temp directory as permitted prefix
resolver := &LocalFolderResolver{
PermittedPrefixes: []string{tempDir},
HomePath: "./",
}
// Test resolving paths
for _, dir := range dirs {
fullPath, err := resolver.LocalPath(filepath.Join(tempDir, dir))
require.NoError(t, err)
require.Equal(t, filepath.Join(tempDir, dir), fullPath)
}
// Test repository with the temp directory
repo := NewLocal(&provisioning.Repository{
Spec: provisioning.RepositorySpec{
Local: &provisioning.LocalRepositoryConfig{
Path: "github",
Path: tempDir,
},
},
}, resolver)
// Full tree
tree, err := r.ReadTree(context.Background(), "")
// Verify we can read the tree
tree, err := repo.ReadTree(context.Background(), "")
require.NoError(t, err)
names := []string{}
for _, v := range tree {
names = append(names, v.Path)
// Collect all paths from the tree
paths := make([]string, 0, len(tree))
for _, item := range tree {
paths = append(paths, item.Path)
}
require.Equal(t, []string{
"client.go",
"factory.go",
"impl.go",
"impl_test.go",
"mock_client.go",
"mock_commit_file.go",
"mock_repository_content.go",
"testdata",
"testdata/webhook-issue_comment-created.json",
"testdata/webhook-ping-check.json",
"testdata/webhook-pull_request-opened.json",
"testdata/webhook-push-different_branch.json",
"testdata/webhook-push-nested.json",
"testdata/webhook-push-nothing_relevant.json",
}, names)
v, err := r.Read(context.Background(), "testdata", "")
// Sort paths for consistent comparison
sort.Strings(paths)
// Verify all directories and files are present
expectedPaths := []string{
"another",
"another/path",
"another/path/file.txt",
"level1",
"level1/file1.txt",
"level1/level2",
"level1/level2/file2.txt",
"level1/level2/level3",
"level1/level2/level3/file3.txt",
"root.txt",
}
require.Equal(t, expectedPaths, paths)
// Test reading a specific file
file, err := repo.Read(context.Background(), "level1/level2/file2.txt", "")
require.NoError(t, err)
require.Equal(t, "testdata", v.Path)
require.Nil(t, v.Data)
v, err = r.Read(context.Background(), "testdata/webhook-push-nested.json", "")
require.NoError(t, err)
require.Equal(t, "4eb879daca9942a887862b3d76fe9f24528d0408", v.Hash)
// read unknown file
_, err = r.Read(context.Background(), "testdata/missing", "")
require.True(t, apierrors.IsNotFound(err)) // 404 error
_, err = r.Read(context.Background(), "testdata/webhook-push-nested.json/", "")
require.Error(t, err) // not a directory
require.Equal(t, "level1/level2/file2.txt", file.Path)
require.Equal(t, []byte("level 2 content"), file.Data)
}
func TestLocal(t *testing.T) {
@@ -162,10 +162,8 @@ type RepositoryWithURLs interface {
type Hooks interface {
Repository
// For repositories that support webhooks
Webhook(ctx context.Context, req *http.Request) (*provisioning.WebhookResponse, error)
OnCreate(ctx context.Context) (*provisioning.WebhookStatus, error)
OnUpdate(ctx context.Context) (*provisioning.WebhookStatus, error)
OnCreate(ctx context.Context) ([]map[string]interface{}, error)
OnUpdate(ctx context.Context) ([]map[string]interface{}, error)
OnDelete(ctx context.Context) error
}
+2 -2
View File
@@ -67,7 +67,7 @@ func (b *APIBuilder) GetAPIRoutes(gv schema.GroupVersion) *builder.APIRoutes {
},
},
},
Handler: withTimeoutFunc(b.handleStats, 30*time.Second),
Handler: WithTimeoutFunc(b.handleStats, 30*time.Second),
},
{
Path: "settings",
@@ -115,7 +115,7 @@ func (b *APIBuilder) GetAPIRoutes(gv schema.GroupVersion) *builder.APIRoutes {
},
},
},
Handler: withTimeoutFunc(b.handleSettings, 30*time.Second),
Handler: WithTimeoutFunc(b.handleSettings, 30*time.Second),
},
},
}
+1 -1
View File
@@ -43,7 +43,7 @@ func (*testConnector) NewConnectOptions() (runtime.Object, bool, string) {
}
func (s *testConnector) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
return withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return WithTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, err := readBody(r, defaultMaxBodySize)
if err != nil {
responder.Error(err)
+5 -5
View File
@@ -6,8 +6,8 @@ import (
"time"
)
// withTimeout adds a timeout context to the request
func withTimeout(h http.Handler, timeout time.Duration) http.Handler {
// WithTimeout adds a timeout context to the request
func WithTimeout(h http.Handler, timeout time.Duration) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), timeout)
defer cancel()
@@ -15,7 +15,7 @@ func withTimeout(h http.Handler, timeout time.Duration) http.Handler {
})
}
// withTimeoutFunc adds a timeout context to the request
func withTimeoutFunc(f func(w http.ResponseWriter, r *http.Request), timeout time.Duration) func(w http.ResponseWriter, r *http.Request) {
return withTimeout(http.HandlerFunc(f), timeout).ServeHTTP
// WithTimeoutFunc adds a timeout context to the request
func WithTimeoutFunc(f func(w http.ResponseWriter, r *http.Request), timeout time.Duration) func(w http.ResponseWriter, r *http.Request) {
return WithTimeout(http.HandlerFunc(f), timeout).ServeHTTP
}
@@ -38,7 +38,7 @@ func TestWithTimeout(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
w := httptest.NewRecorder()
r := httptest.NewRequest("GET", "/", nil)
withTimeout(handler, tt.timeout).ServeHTTP(w, r)
WithTimeout(handler, tt.timeout).ServeHTTP(w, r)
if w.Code != tt.wantStatus {
t.Errorf("withTimeout() status = %v, want %v", w.Code, tt.wantStatus)
@@ -0,0 +1,188 @@
package webhooks
import (
"context"
"fmt"
"path/filepath"
"strings"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
provisioningapis "github.com/grafana/grafana/pkg/registry/apis/provisioning"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository/github"
gogit "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository/go-git"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/secrets"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/webhooks/pullrequest"
"github.com/grafana/grafana/pkg/services/apiserver"
"github.com/grafana/grafana/pkg/services/rendering"
grafanasecrets "github.com/grafana/grafana/pkg/services/secrets"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/storage/unified/resource"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/kube-openapi/pkg/spec3"
)
// WebhookExtraBuilder is a function that returns an ExtraBuilder.
// It is used to add additional functionality for webhooks
type WebhookExtraBuilder struct {
// HACK: We need to wrap the builder to please wire so that it can uniquely identify the dependency
provisioningapis.ExtraBuilder
}
func ProvideWebhooks(
cfg *setting.Cfg,
// FIXME: use multi-tenant service when one exists. In this state, we can't make this a multi-tenant service!
secretsSvc grafanasecrets.Service,
ghFactory *github.Factory,
renderer rendering.Service,
blobstore resource.ResourceClient,
configProvider apiserver.RestConfigProvider,
) WebhookExtraBuilder {
return WebhookExtraBuilder{
ExtraBuilder: func(b *provisioningapis.APIBuilder) provisioningapis.Extra {
urlProvider := func(_ string) string {
return cfg.AppURL
}
// HACK: Assume is only public if it is HTTPS
isPublic := strings.HasPrefix(urlProvider(""), "https://")
clients := resources.NewClientFactory(configProvider)
parsers := resources.NewParserFactory(clients)
screenshotRenderer := pullrequest.NewScreenshotRenderer(renderer, blobstore)
render := NewRenderConnector(blobstore, b)
webhook := NewWebhookConnector(
isPublic,
b,
screenshotRenderer,
)
evaluator := pullrequest.NewEvaluator(screenshotRenderer, parsers, urlProvider)
commenter := pullrequest.NewCommenter()
pullRequestWorker := pullrequest.NewPullRequestWorker(evaluator, commenter)
return NewWebhookExtra(
render,
webhook,
urlProvider,
secrets.NewSingleTenant(secretsSvc),
ghFactory,
filepath.Join(cfg.DataPath, "clone"),
parsers,
[]jobs.Worker{pullRequestWorker},
)
},
}
}
// WebhookExtra implements the Extra interface for webhooks
// to wrap around
type WebhookExtra struct {
render *renderConnector
webhook *webhookConnector
urlProvider func(namespace string) string
secrets secrets.Service
ghFactory *github.Factory
clonedir string
parsers resources.ParserFactory
workers []jobs.Worker
}
func NewWebhookExtra(
render *renderConnector,
webhook *webhookConnector,
urlProvider func(namespace string) string,
secrets secrets.Service,
ghFactory *github.Factory,
clonedir string,
parsers resources.ParserFactory,
workers []jobs.Worker,
) *WebhookExtra {
return &WebhookExtra{
render: render,
webhook: webhook,
urlProvider: urlProvider,
secrets: secrets,
ghFactory: ghFactory,
clonedir: clonedir,
parsers: parsers,
workers: workers,
}
}
// Authorize delegates authorization to the webhook connector
func (e *WebhookExtra) Authorize(ctx context.Context, a authorizer.Attributes) (decision authorizer.Decision, reason string, err error) {
webhookDecision, webhookReason, webhookErr := e.webhook.Authorize(ctx, a)
if webhookDecision != authorizer.DecisionNoOpinion {
return webhookDecision, webhookReason, webhookErr
}
return e.render.Authorize(ctx, a)
}
// Mutate delegates mutation to the webhook connector
func (e *WebhookExtra) Mutate(ctx context.Context, r *provisioning.Repository) error {
// Encrypt webhook secret if present
if r.Status.Webhook != nil && r.Status.Webhook.Secret != "" {
encryptedSecret, err := e.secrets.Encrypt(ctx, []byte(r.Status.Webhook.Secret))
if err != nil {
return fmt.Errorf("failed to encrypt webhook secret: %w", err)
}
r.Status.Webhook.EncryptedSecret = encryptedSecret
r.Status.Webhook.Secret = ""
}
return nil
}
// UpdateStorage updates the storage with both render and webhook connectors
func (e *WebhookExtra) UpdateStorage(storage map[string]rest.Storage) error {
if err := e.webhook.UpdateStorage(storage); err != nil {
return err
}
return e.render.UpdateStorage(storage)
}
// PostProcessOpenAPI processes OpenAPI specs for both connectors
func (e *WebhookExtra) PostProcessOpenAPI(oas *spec3.OpenAPI) error {
if err := e.webhook.PostProcessOpenAPI(oas); err != nil {
return err
}
return e.render.PostProcessOpenAPI(oas)
}
// GetJobWorkers returns job workers from the webhook connector
func (e *WebhookExtra) GetJobWorkers() []jobs.Worker {
return e.workers
}
// AsRepository delegates repository creation to the webhook connector
func (e *WebhookExtra) AsRepository(ctx context.Context, r *provisioning.Repository) (repository.Repository, error) {
if r.Spec.Type == provisioning.GitHubRepositoryType {
gvr := provisioning.RepositoryResourceInfo.GroupVersionResource()
webhookURL := fmt.Sprintf(
"%sapis/%s/%s/namespaces/%s/%s/%s/webhook",
e.urlProvider(r.GetNamespace()),
gvr.Group,
gvr.Version,
r.GetNamespace(),
gvr.Resource,
r.GetName(),
)
cloneFn := func(ctx context.Context, opts repository.CloneOptions) (repository.ClonedRepository, error) {
return gogit.Clone(ctx, e.clonedir, r, opts, e.secrets)
}
basicRepo, err := repository.NewGitHub(ctx, r, e.ghFactory, e.secrets, cloneFn)
if err != nil {
return nil, err
}
return NewGithubWebhookRepository(basicRepo, webhookURL, e.secrets), nil
}
return nil, nil
}
@@ -1,4 +1,4 @@
package provisioning
package webhooks
import (
"context"
@@ -10,16 +10,27 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/kube-openapi/pkg/spec3"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
provisioningapis "github.com/grafana/grafana/pkg/registry/apis/provisioning"
"github.com/grafana/grafana/pkg/storage/unified/resource"
)
type renderConnector struct {
blob resource.BlobStoreClient
unified resource.ResourceClient
core *provisioningapis.APIBuilder
}
func NewRenderConnector(unified resource.ResourceClient, core *provisioningapis.APIBuilder) *renderConnector {
return &renderConnector{
unified: unified,
core: core,
}
}
func (*renderConnector) New() runtime.Object {
@@ -44,6 +55,58 @@ func (*renderConnector) NewConnectOptions() (runtime.Object, bool, string) {
return nil, true, ""
}
func (c *renderConnector) Authorize(_ context.Context, a authorizer.Attributes) (decision authorizer.Decision, reason string, err error) {
if a.GetResource() == provisioning.RepositoryResourceInfo.GetName() && a.GetSubresource() == "render" {
return authorizer.DecisionAllow, "", nil
}
return authorizer.DecisionNoOpinion, "", nil
}
func (c *renderConnector) PostProcessOpenAPI(oas *spec3.OpenAPI) error {
root := "/apis/" + c.core.GetGroupVersion().String() + "/"
repoprefix := root + "namespaces/{namespace}/repositories/{name}"
delete(oas.Paths.Paths, repoprefix+"/render")
sub := oas.Paths.Paths[repoprefix+"/render/{path}"]
if sub != nil {
sub.Get.Description = "get a rendered preview image"
sub.Get.Responses = &spec3.Responses{
ResponsesProps: spec3.ResponsesProps{
StatusCodeResponses: map[int]*spec3.Response{
200: {
ResponseProps: spec3.ResponseProps{
Content: map[string]*spec3.MediaType{
"image/png": {},
},
Description: "OK",
},
},
},
},
}
// Replace {path} with {guid} (it is a GUID, but all k8s sub-resources are called path)
for _, v := range sub.Parameters {
if v.Name == "path" {
v.Name = "guid"
v.Description = "Image GUID"
break
}
}
delete(oas.Paths.Paths, repoprefix+"/render/{path}")
oas.Paths.Paths[repoprefix+"/render/{guid}"] = sub
}
return nil
}
func (c *renderConnector) UpdateStorage(storage map[string]rest.Storage) error {
storage[provisioning.RepositoryResourceInfo.StoragePath("render")] = c
return nil
}
func (c *renderConnector) Connect(
ctx context.Context,
name string,
@@ -51,7 +114,7 @@ func (c *renderConnector) Connect(
responder rest.Responder,
) (http.Handler, error) {
namespace := request.NamespaceValue(ctx)
return withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return provisioningapis.WithTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
prefix := fmt.Sprintf("/%s/render", name)
idx := strings.Index(r.URL.Path, prefix)
if idx == -1 {
@@ -64,12 +127,12 @@ func (c *renderConnector) Connect(
responder.Error(apierrors.NewNotFound(provisioning.RepositoryResourceInfo.GroupResource(), "render"))
return
}
if !validBlobID(blobID) {
if !provisioningapis.ValidUUID(blobID) {
responder.Error(apierrors.NewBadRequest(fmt.Sprintf("invalid blob id: %s", blobID)))
return
}
rsp, err := c.blob.GetBlob(ctx, &resource.GetBlobRequest{
rsp, err := c.unified.GetBlob(ctx, &resource.GetBlobRequest{
Resource: &resource.ResourceKey{
Namespace: namespace,
Group: provisioning.GROUP,
@@ -108,22 +171,6 @@ func (c *renderConnector) Connect(
}), 20*time.Second), nil
}
// validBlobID ensures the ID is valid for a blob.
// The ID is always a UUID. As such, this checks for something that can resemble a UUID.
// This does not check for the ID to be an actual UUID, as the blob store may change their ID format, which we do not wish to stand in the way of.
func validBlobID(id string) bool {
for _, c := range id {
// [a-zA-Z0-9\-] are valid characters.
az := c >= 'a' && c <= 'z'
AZ := c >= 'A' && c <= 'Z'
digit := c >= '0' && c <= '9'
if !az && !AZ && !digit && c != '-' {
return false
}
}
return true
}
var (
_ rest.Connecter = (*renderConnector)(nil)
_ rest.Storage = (*renderConnector)(nil)
@@ -0,0 +1,360 @@
package webhooks
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"slices"
"github.com/google/go-github/v70/github"
"github.com/google/uuid"
"github.com/grafana/grafana-app-sdk/logging"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
pgh "github.com/grafana/grafana/pkg/registry/apis/provisioning/repository/github"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/secrets"
apierrors "k8s.io/apimachinery/pkg/api/errors"
)
var subscribedEvents = []string{"push", "pull_request"}
type WebhookRepository interface {
Webhook(ctx context.Context, req *http.Request) (*provisioning.WebhookResponse, error)
}
type GithubWebhookRepository interface {
repository.GithubRepository
repository.Hooks
WebhookRepository
}
type githubWebhookRepository struct {
repository.GithubRepository
config *provisioning.Repository
owner string
repo string
secrets secrets.Service
gh pgh.Client
webhookURL string
}
func NewGithubWebhookRepository(
basic repository.GithubRepository,
webhookURL string,
secrets secrets.Service,
) GithubWebhookRepository {
return &githubWebhookRepository{
GithubRepository: basic,
config: basic.Config(),
owner: basic.Owner(),
repo: basic.Repo(),
gh: basic.Client(),
webhookURL: webhookURL,
secrets: secrets,
}
}
// Webhook implements Repository.
func (r *githubWebhookRepository) Webhook(ctx context.Context, req *http.Request) (*provisioning.WebhookResponse, error) {
if r.config.Status.Webhook == nil {
return nil, fmt.Errorf("unexpected webhook request")
}
secret, err := r.secrets.Decrypt(ctx, r.config.Status.Webhook.EncryptedSecret)
if err != nil {
return nil, fmt.Errorf("failed to decrypt secret: %w", err)
}
payload, err := github.ValidatePayload(req, secret)
if err != nil {
return nil, apierrors.NewUnauthorized("invalid signature")
}
return r.parseWebhook(github.WebHookType(req), payload)
}
// This method does not include context because it does delegate any more requests
func (r *githubWebhookRepository) parseWebhook(messageType string, payload []byte) (*provisioning.WebhookResponse, error) {
event, err := github.ParseWebHook(messageType, payload)
if err != nil {
return nil, apierrors.NewBadRequest("invalid payload")
}
switch event := event.(type) {
case *github.PushEvent:
return r.parsePushEvent(event)
case *github.PullRequestEvent:
return r.parsePullRequestEvent(event)
case *github.PingEvent:
return &provisioning.WebhookResponse{
Code: http.StatusOK,
Message: "ping received",
}, nil
default:
return &provisioning.WebhookResponse{
Code: http.StatusNotImplemented,
Message: fmt.Sprintf("unsupported messageType: %s", messageType),
}, nil
}
}
func (r *githubWebhookRepository) parsePushEvent(event *github.PushEvent) (*provisioning.WebhookResponse, error) {
if event.GetRepo() == nil {
return nil, fmt.Errorf("missing repository in push event")
}
if event.GetRepo().GetFullName() != fmt.Sprintf("%s/%s", r.owner, r.repo) {
return nil, fmt.Errorf("repository mismatch")
}
// No need to sync if not enabled
if !r.config.Spec.Sync.Enabled {
return &provisioning.WebhookResponse{Code: http.StatusOK}, nil
}
// Skip silently if the event is not for the main/master branch
// as we cannot configure the webhook to only publish events for the main branch
if event.GetRef() != fmt.Sprintf("refs/heads/%s", r.config.Spec.GitHub.Branch) {
return &provisioning.WebhookResponse{Code: http.StatusOK}, nil
}
return &provisioning.WebhookResponse{
Code: http.StatusAccepted,
Job: &provisioning.JobSpec{
Repository: r.config.GetName(),
Action: provisioning.JobActionPull,
Pull: &provisioning.SyncJobOptions{
Incremental: true,
},
},
}, nil
}
func (r *githubWebhookRepository) parsePullRequestEvent(event *github.PullRequestEvent) (*provisioning.WebhookResponse, error) {
if event.GetRepo() == nil {
return nil, fmt.Errorf("missing repository in pull request event")
}
cfg := r.config.Spec.GitHub
if cfg == nil {
return nil, fmt.Errorf("missing GitHub config")
}
if event.GetRepo().GetFullName() != fmt.Sprintf("%s/%s", r.owner, r.repo) {
return nil, fmt.Errorf("repository mismatch")
}
pr := event.GetPullRequest()
if pr == nil {
return nil, fmt.Errorf("expected PR in event")
}
if pr.GetBase().GetRef() != r.config.Spec.GitHub.Branch {
return &provisioning.WebhookResponse{
Code: http.StatusOK,
Message: fmt.Sprintf("ignoring pull request event as %s is not the configured branch", pr.GetBase().GetRef()),
}, nil
}
action := event.GetAction()
if action != "opened" && action != "reopened" && action != "synchronize" {
return &provisioning.WebhookResponse{
Code: http.StatusOK, // Nothing needed
Message: fmt.Sprintf("ignore pull request event: %s", action),
}, nil
}
// Queue an async job that will parse files
return &provisioning.WebhookResponse{
Code: http.StatusAccepted, // Nothing needed
Message: fmt.Sprintf("pull request: %s", action),
Job: &provisioning.JobSpec{
Repository: r.config.GetName(),
Action: provisioning.JobActionPullRequest,
PullRequest: &provisioning.PullRequestJobOptions{
URL: pr.GetHTMLURL(),
PR: pr.GetNumber(),
Ref: pr.GetHead().GetRef(),
Hash: pr.GetHead().GetSHA(),
},
},
}, nil
}
// CommentPullRequest adds a comment to a pull request.
func (r *githubWebhookRepository) CommentPullRequest(ctx context.Context, prNumber int, comment string) error {
ctx, _ = r.logger(ctx, "")
return r.gh.CreatePullRequestComment(ctx, r.owner, r.repo, prNumber, comment)
}
func (r *githubWebhookRepository) createWebhook(ctx context.Context) (pgh.WebhookConfig, error) {
secret, err := uuid.NewRandom()
if err != nil {
return pgh.WebhookConfig{}, fmt.Errorf("could not generate secret: %w", err)
}
cfg := pgh.WebhookConfig{
URL: r.webhookURL,
Secret: secret.String(),
ContentType: "json",
Events: subscribedEvents,
Active: true,
}
hook, err := r.gh.CreateWebhook(ctx, r.owner, r.repo, cfg)
if err != nil {
return pgh.WebhookConfig{}, err
}
// HACK: GitHub does not return the secret, so we need to update it manually
hook.Secret = cfg.Secret
logging.FromContext(ctx).Info("webhook created", "url", cfg.URL, "id", hook.ID)
return hook, nil
}
// updateWebhook checks if the webhook needs to be updated and updates it if necessary.
// if the webhook does not exist, it will create it.
func (r *githubWebhookRepository) updateWebhook(ctx context.Context) (pgh.WebhookConfig, bool, error) {
if r.config.Status.Webhook == nil || r.config.Status.Webhook.ID == 0 {
hook, err := r.createWebhook(ctx)
if err != nil {
return pgh.WebhookConfig{}, false, err
}
return hook, true, nil
}
hook, err := r.gh.GetWebhook(ctx, r.owner, r.repo, r.config.Status.Webhook.ID)
switch {
case errors.Is(err, pgh.ErrResourceNotFound):
hook, err := r.createWebhook(ctx)
if err != nil {
return pgh.WebhookConfig{}, false, err
}
return hook, true, nil
case err != nil:
return pgh.WebhookConfig{}, false, fmt.Errorf("get webhook: %w", err)
}
hook.Secret = r.config.Status.Webhook.Secret // we always random gen this, so don't use it for mustUpdate below.
var mustUpdate bool
if hook.URL != r.webhookURL {
mustUpdate = true
hook.URL = r.webhookURL
}
if !slices.Equal(hook.Events, subscribedEvents) {
mustUpdate = true
hook.Events = subscribedEvents
}
if !mustUpdate {
return hook, false, nil
}
// Something has changed in the webhook. Let's rotate the secret as well, so as to ensure we end up with a 100% correct webhook.
secret, err := uuid.NewRandom()
if err != nil {
return pgh.WebhookConfig{}, false, fmt.Errorf("could not generate secret: %w", err)
}
hook.Secret = secret.String()
if err := r.gh.EditWebhook(ctx, r.owner, r.repo, hook); err != nil {
return pgh.WebhookConfig{}, false, fmt.Errorf("edit webhook: %w", err)
}
return hook, true, nil
}
func (r *githubWebhookRepository) deleteWebhook(ctx context.Context) error {
if r.config.Status.Webhook == nil {
return fmt.Errorf("webhook not found")
}
id := r.config.Status.Webhook.ID
if err := r.gh.DeleteWebhook(ctx, r.owner, r.repo, id); err != nil {
return fmt.Errorf("delete webhook: %w", err)
}
logging.FromContext(ctx).Info("webhook deleted", "url", r.config.Status.Webhook.URL, "id", id)
return nil
}
func (r *githubWebhookRepository) OnCreate(ctx context.Context) ([]map[string]interface{}, error) {
if len(r.webhookURL) == 0 {
return nil, nil
}
ctx, _ = r.logger(ctx, "")
hook, err := r.createWebhook(ctx)
if err != nil {
return nil, err
}
return []map[string]interface{}{
{
"op": "replace",
"path": "/status/webhook",
"value": &provisioning.WebhookStatus{
ID: hook.ID,
URL: hook.URL,
Secret: hook.Secret,
SubscribedEvents: hook.Events,
},
},
}, nil
}
func (r *githubWebhookRepository) OnUpdate(ctx context.Context) ([]map[string]interface{}, error) {
if len(r.webhookURL) == 0 {
return nil, nil
}
ctx, _ = r.logger(ctx, "")
hook, _, err := r.updateWebhook(ctx)
if err != nil {
return nil, err
}
return []map[string]interface{}{
{
"op": "replace",
"path": "/status/webhook",
"value": &provisioning.WebhookStatus{
ID: hook.ID,
URL: hook.URL,
Secret: hook.Secret,
SubscribedEvents: hook.Events,
},
},
}, nil
}
func (r *githubWebhookRepository) OnDelete(ctx context.Context) error {
if len(r.webhookURL) == 0 {
return nil
}
ctx, _ = r.logger(ctx, "")
return r.deleteWebhook(ctx)
}
func (r *githubWebhookRepository) logger(ctx context.Context, ref string) (context.Context, logging.Logger) {
logger := logging.FromContext(ctx)
type containsGh int
var containsGhKey containsGh
if ctx.Value(containsGhKey) != nil {
return ctx, logging.FromContext(ctx)
}
if ref == "" {
ref = r.config.Spec.GitHub.Branch
}
logger = logger.With(slog.Group("github_repository", "owner", r.owner, "name", r.repo, "ref", ref))
ctx = logging.Context(ctx, logger)
// We want to ensure we don't add multiple github_repository keys. With doesn't deduplicate the keys...
ctx = context.WithValue(ctx, containsGhKey, true)
return ctx, logger
}
File diff suppressed because it is too large Load Diff
@@ -1,24 +1,24 @@
package provisioning
package webhooks
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/kube-openapi/pkg/spec3"
"github.com/grafana/grafana-app-sdk/logging"
"github.com/grafana/grafana/pkg/apimachinery/identity"
provisioning "github.com/grafana/grafana/pkg/apis/provisioning/v0alpha1"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
provisioningapis "github.com/grafana/grafana/pkg/registry/apis/provisioning"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/repository"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/webhooks/pullrequest"
)
// Webhook endpoint max size (25MB)
@@ -27,18 +27,21 @@ const webhookMaxBodySize = 25 * 1024 * 1024
// This only works for github right now
type webhookConnector struct {
client ClientGetter
getter RepoGetter
jobs jobs.Queue
webhooksEnabled bool
core *provisioningapis.APIBuilder
renderer pullrequest.ScreenshotRenderer
}
func NewWebhookConnector(client ClientGetter, getter RepoGetter, jobs jobs.Queue, webhooksEnabled bool) *webhookConnector {
func NewWebhookConnector(
webhooksEnabled bool,
// TODO: use interface for this
core *provisioningapis.APIBuilder,
renderer pullrequest.ScreenshotRenderer,
) *webhookConnector {
return &webhookConnector{
client: client,
getter: getter,
jobs: jobs,
webhooksEnabled: webhooksEnabled,
core: core,
renderer: renderer,
}
}
@@ -67,6 +70,33 @@ func (*webhookConnector) NewConnectOptions() (runtime.Object, bool, string) {
return nil, false, ""
}
func (s *webhookConnector) Authorize(ctx context.Context, a authorizer.Attributes) (decision authorizer.Decision, reason string, err error) {
if provisioning.RepositoryResourceInfo.GetName() == a.GetResource() && a.GetSubresource() == "webhook" {
// When the resource is a webhook, we'll deal with permissions manually by checking signatures or similar in the webhook handler.
// The user in this context is usually an anonymous user, but may also be an authenticated synthetic check by the Grafana instance's operator as well.
// For context on the anonymous user, check the authn/clients/provisioning.go file.
return authorizer.DecisionAllow, "", nil
}
return authorizer.DecisionNoOpinion, "", nil
}
func (s *webhookConnector) UpdateStorage(storage map[string]rest.Storage) error {
storage[provisioning.RepositoryResourceInfo.StoragePath("webhook")] = s
return nil
}
func (s *webhookConnector) PostProcessOpenAPI(oas *spec3.OpenAPI) error {
root := "/apis/" + s.core.GetGroupVersion().String() + "/"
repoprefix := root + "namespaces/{namespace}/repositories/{name}"
sub := oas.Paths.Paths[repoprefix+"/webhook"]
if sub != nil && sub.Get != nil {
sub.Post.Description = "Currently only supports github webhooks"
}
return nil
}
func (s *webhookConnector) Connect(ctx context.Context, name string, opts runtime.Object, responder rest.Responder) (http.Handler, error) {
namespace := request.NamespaceValue(ctx)
ctx, _, err := identity.WithProvisioningIdentity(ctx, namespace)
@@ -75,12 +105,12 @@ func (s *webhookConnector) Connect(ctx context.Context, name string, opts runtim
}
// Get the repository with the worker identity (since the request user is likely anonymous)
repo, err := s.getter.GetHealthyRepository(ctx, name)
repo, err := s.core.GetRepository(ctx, name)
if err != nil {
return nil, err
}
return withTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return provisioningapis.WithTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
logger := logging.FromContext(r.Context()).With("logger", "webhook-connector", "repo", name)
ctx := logging.Context(r.Context(), logger)
if !s.webhooksEnabled {
@@ -88,7 +118,7 @@ func (s *webhookConnector) Connect(ctx context.Context, name string, opts runtim
return
}
hooks, ok := repo.(repository.Hooks)
hooks, ok := repo.(WebhookRepository)
if !ok {
responder.Error(errors.NewBadRequest("the repository does not support webhooks"))
return
@@ -108,14 +138,14 @@ func (s *webhookConnector) Connect(ctx context.Context, name string, opts runtim
return
}
if err := s.updateLastEvent(ctx, repo, name, namespace); err != nil {
if err := s.updateLastEvent(ctx, repo); err != nil {
// Continue processing as this is non-critical; the update is purely informational
logger.Error("failed to update last event", "error", err)
}
if rsp.Job != nil {
rsp.Job.Repository = name
job, err := s.jobs.Insert(ctx, namespace, *rsp.Job)
job, err := s.core.GetJobQueue().Insert(ctx, namespace, *rsp.Job)
if err != nil {
responder.Error(err)
return
@@ -131,11 +161,11 @@ func (s *webhookConnector) Connect(ctx context.Context, name string, opts runtim
// updateLastEvent updates the last event time for the webhook
// This is to provide some visibility that the webhook is still active and working
// It's not a good idea to update the webhook status too often, so we only update it if it's been a while
func (s *webhookConnector) updateLastEvent(ctx context.Context, repo repository.Repository, name, namespace string) error {
client := s.client.GetClient()
if client == nil {
func (s *webhookConnector) updateLastEvent(ctx context.Context, repo repository.Repository) error {
patcher := s.core.GetStatusPatcher()
if patcher == nil {
// This would only happen if we wired things up incorrectly
return fmt.Errorf("client is nil")
return fmt.Errorf("status patcher is nil")
}
lastEvent := time.UnixMilli(repo.Config().Status.Webhook.LastEvent)
@@ -148,13 +178,7 @@ func (s *webhookConnector) updateLastEvent(ctx context.Context, repo repository.
"value": time.Now().UnixMilli(),
}
patch, err := json.Marshal([]map[string]interface{}{patchOp})
if err != nil {
return fmt.Errorf("marshal patch: %w", err)
}
if _, err = client.Repositories(namespace).
Patch(ctx, name, types.JSONPatchType, patch, metav1.PatchOptions{}, "status"); err != nil {
if err := patcher.Patch(ctx, repo.Config(), patchOp); err != nil {
return fmt.Errorf("patch status: %w", err)
}
}
+14
View File
@@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/registry/apis/folders"
"github.com/grafana/grafana/pkg/registry/apis/iam"
"github.com/grafana/grafana/pkg/registry/apis/provisioning"
"github.com/grafana/grafana/pkg/registry/apis/provisioning/webhooks"
"github.com/grafana/grafana/pkg/registry/apis/query"
"github.com/grafana/grafana/pkg/registry/apis/secret"
"github.com/grafana/grafana/pkg/registry/apis/service"
@@ -17,6 +18,18 @@ import (
"github.com/grafana/grafana/pkg/services/pluginsintegration/plugincontext"
)
// HACK: This is a hack so that wire can uniquely identify dependencies
func MergeProvisioningExtras(webhook webhooks.WebhookExtraBuilder) []provisioning.ExtraBuilder {
return []provisioning.ExtraBuilder{
webhook.ExtraBuilder,
}
}
var ProvisioningExtras = wire.NewSet(
webhooks.ProvideWebhooks,
MergeProvisioningExtras,
)
var WireSet = wire.NewSet(
ProvideRegistryServiceSink, // dummy background service that forces registration
@@ -32,6 +45,7 @@ var WireSet = wire.NewSet(
datasource.RegisterAPIService,
folders.RegisterAPIService,
iam.RegisterAPIService,
ProvisioningExtras,
provisioning.RegisterAPIService,
service.RegisterAPIService,
query.RegisterAPIService,