CI: Report to bench (#101686)

* feat: report to bench

* fix: output json

* chore: own the code

* temp: limit files to test for faster iteration

* temp: force bench to run for testing purposes

* fix: bench has been merged to main

* fix: correct file name

* temp: remove temp

* feat: pin the tparse binary

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

---------

Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>

.github/CODEOWNERS

@@ -756,6 +756,7 @@ embed.go @grafana/grafana-as-code
 /.github/pr-commands.json @tolzhabayev
 /.github/renovate.json5 @grafana/frontend-ops
 /.github/actions/test-coverage-processor/action.yml @grafana/grafana-backend-group
+/.github/actions/setup-grafana-bench/ @Proximyst
 /.github/workflows/add-to-whats-new.yml @grafana/docs-tooling
 /.github/workflows/auto-triager/ @grafana/plugins-platform-frontend
 /.github/workflows/alerting-swagger-gen.yml @grafana/alerting-backend

.github/actions/setup-grafana-bench/action.yml

@@ -0,0 +1,44 @@
+name: 'Setup Grafana Bench'
+description: 'Sets up and installs Grafana Bench'
+
+inputs:
+  github-app-name:
+    description: 'Name of the GitHub App in Vault'
+    required: false
+    default: 'grafana-ci-bot'
+  branch:
+    description: 'The branch to install from'
+    required: false
+    default: 'main'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Retrieve GitHub App secrets
+      id: get-secrets
+      uses: grafana/shared-workflows/actions/get-vault-secrets@get-vault-secrets-v1.0.1
+      with:
+        repo_secrets: |
+          APP_ID=${{ inputs.github-app-name }}:app-id
+          APP_INSTALLATION_ID=${{ inputs.github-app-name }}:app-installation-id
+          PRIVATE_KEY=${{ inputs.github-app-name }}:private-key
+
+    - name: Generate GitHub App token
+      id: generate_token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ env.APP_ID }}
+        private-key: ${{ env.PRIVATE_KEY }}
+        repositories: "grafana-bench"
+        owner: "grafana"
+
+    - name: Setup Bench
+      shell: bash
+      env:
+        GH_TOKEN: ${{ steps.generate_token.outputs.token }}
+      run: |
+        git clone https://x-access-token:${GH_TOKEN}@github.com/grafana/grafana-bench.git ../grafana-bench
+
+        cd ../grafana-bench
+        git switch ${{ inputs.branch }}
+        go install .

.github/workflows/pr-backend-unit-tests.yml

@@ -39,6 +39,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y build-essential shared-mime-info
+          go install github.com/mfridman/tparse@c1754a1f484ac5cd422697b0fec635177ddc8507 # v0.17.0
 
       - name: Verify code generation
         run: |
@@ -49,7 +50,7 @@ jobs:
         run: make gen-go
 
       - name: Run unit tests
-        run: COVER_OPTS="-coverprofile=be-unit.cov -coverpkg=github.com/grafana/grafana/..." make test-go-unit
+        run: COVER_OPTS="-coverprofile=be-unit.cov -coverpkg=github.com/grafana/grafana/..." GO_TEST_OUTPUT="/tmp/unit.log" make test-go-unit
 
       - name: Process and upload coverage
         uses: ./.github/actions/test-coverage-processor
@@ -61,6 +62,22 @@ jobs:
           codecov-flag: 'be-unit'
           codecov-name: 'be-unit'
 
+      - name: Install Grafana Bench
+        # We can't allow forks here, as we need secret access.
+        if: ${{ github.event_name != 'pull_request' }}
+        uses: ./.github/actions/setup-grafana-bench
+
+      - name: Process output for Bench
+        if: ${{ github.event_name != 'pull_request' }}
+        run: |
+          grafana-bench report \
+            --trigger pr-backend-unit-tests-oss \
+            --report-input go \
+            --report-output log \
+            --grafana-version "$(git rev-parse HEAD)" \
+            --suite-name grafana-oss-unit-tests \
+            /tmp/unit.log
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true

Makefile

@@ -17,6 +17,7 @@ GO_RACE_FLAG := $(if $(GO_RACE),-race)
 GO_BUILD_FLAGS += $(if $(GO_BUILD_DEV),-dev)
 GO_BUILD_FLAGS += $(if $(GO_BUILD_TAGS),-build-tags=$(GO_BUILD_TAGS))
 GO_BUILD_FLAGS += $(GO_RACE_FLAG)
+GO_TEST_OUTPUT := $(shell [ -n "$(GO_TEST_OUTPUT)" ] && echo '-json | tee $(GO_TEST_OUTPUT) | tparse -all')
 GIT_BASE = remotes/origin/main
 
 # GNU xargs has flag -r, and BSD xargs (e.g. MacOS) has that behaviour by default
@@ -248,7 +249,7 @@ test-go: test-go-unit test-go-integration
 .PHONY: test-go-unit
 test-go-unit: ## Run unit tests for backend with flags.
 	@echo "test backend unit tests"
-	$(GO) test $(GO_RACE_FLAG) -short -covermode=atomic -coverprofile=unit.cov -timeout=30m $(GO_TEST_FILES)
+	$(GO) test $(GO_RACE_FLAG) -short -covermode=atomic -coverprofile=unit.cov -timeout=30m $(GO_TEST_FILES) $(GO_TEST_OUTPUT)
 
 .PHONY: test-go-unit-pretty
 test-go-unit-pretty: check-tparse
@@ -261,7 +262,7 @@ test-go-unit-pretty: check-tparse
 .PHONY: test-go-integration
 test-go-integration: ## Run integration tests for backend with flags.
 	@echo "test backend integration tests"
-	$(GO) test $(GO_RACE_FLAG) -count=1 -run "^TestIntegration" -covermode=atomic -coverprofile=integration.cov -timeout=5m $(GO_INTEGRATION_TESTS)
+	$(GO) test $(GO_RACE_FLAG) -count=1 -run "^TestIntegration" -covermode=atomic -coverprofile=integration.cov -timeout=5m $(GO_INTEGRATION_TESTS) $(GO_TEST_OUTPUT)
 
 .PHONY: test-go-integration-alertmanager
 test-go-integration-alertmanager: ## Run integration tests for the remote alertmanager (config taken from the mimir_backend block).