From 7f55ba9c6ee8db2d1e2524a3b3c4e7ed0511f056 Mon Sep 17 00:00:00 2001 From: Dimitris Sotirakis Date: Wed, 28 Jun 2023 11:12:47 +0300 Subject: [PATCH] CI: Cleanup - Remove security related steps (#70788) * Remove security related steps * More cleanup --- .drone.star | 4 - .drone.yml | 203 ++------------------ scripts/drone/events/release.star | 39 +--- scripts/drone/pipelines/build.star | 3 - scripts/drone/pipelines/publish_images.star | 30 +-- scripts/drone/steps/lib.star | 25 +-- scripts/drone/vault.star | 15 -- 7 files changed, 24 insertions(+), 295 deletions(-) diff --git a/.drone.star b/.drone.star index bc9630209a3..25c64c32e8a 100644 --- a/.drone.star +++ b/.drone.star @@ -11,7 +11,6 @@ load("scripts/drone/events/pr.star", "pr_pipelines") load("scripts/drone/events/main.star", "main_pipelines") load( "scripts/drone/events/release.star", - "artifacts_page_pipeline", "enterprise2_pipelines", "enterprise_pipelines", "integration_test_pipelines", @@ -28,7 +27,6 @@ load( load( "scripts/drone/pipelines/publish_images.star", "publish_image_pipelines_public", - "publish_image_pipelines_security", ) load( "scripts/drone/pipelines/ci_images.star", @@ -52,7 +50,6 @@ def main(_ctx): enterprise_pipelines() + enterprise2_pipelines() + publish_image_pipelines_public() + - publish_image_pipelines_security() + publish_github_pipeline("public") + publish_github_pipeline("security") + publish_aws_marketplace_pipeline("public") + @@ -66,7 +63,6 @@ def main(_ctx): "event": ["promote"], "target": ["test-windows"], }, "oss", "testing")] + - artifacts_page_pipeline() + version_branch_pipelines() + integration_test_pipelines() + publish_ci_windows_test_image_pipeline() + diff --git a/.drone.yml b/.drone.yml index 3eab3feedc9..560ab1db123 100644 --- a/.drone.yml +++ b/.drone.yml @@ -3888,91 +3888,6 @@ volumes: clone: retries: 3 depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: publish-docker-enterprise-security -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - echo $DRONE_RUNNER_NAME - image: alpine:3.17.1 - name: identify-runner -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: [] - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - ./bin/build artifacts docker fetch --edition enterprise - depends_on: - - compile-build-cmd - environment: - DOCKER_ENTERPRISE2_REPO: - from_secret: docker_enterprise2_repo - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key - image: google/cloud-sdk:431.0.0 - name: fetch-images-enterprise - volumes: - - name: docker - path: /var/run/docker.sock -- commands: - - ./bin/grabpl artifacts docker publish --security --dockerhub-repo grafana/grafana-enterprise - --version-tag ${DRONE_TAG} - depends_on: - - fetch-images-enterprise - environment: - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USER: - from_secret: docker_username - GCP_KEY: - from_secret: gcp_key - GITHUB_APP_ID: - from_secret: delivery-bot-app-id - GITHUB_APP_INSTALLATION_ID: - from_secret: delivery-bot-app-installation-id - GITHUB_APP_PRIVATE_KEY: - from_secret: delivery-bot-app-private-key - image: google/cloud-sdk:431.0.0 - name: publish-images-grafana-enterprise - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - event: - - promote - target: - - security -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -clone: - retries: 3 -depends_on: [] environment: EDITION: enterprise2 image_pull_secrets: @@ -4172,18 +4087,14 @@ steps: image: golang:1.20.4 name: compile-build-cmd - commands: - - ./bin/build artifacts packages --security --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} + - ./bin/build artifacts packages --tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}} depends_on: - compile-build-cmd environment: - ENTERPRISE2_SECURITY_PREFIX: - from_secret: enterprise2_security_prefix GCP_KEY: from_secret: gcp_key PRERELEASE_BUCKET: from_secret: prerelease_bucket - SECURITY_DEST_BUCKET: - from_secret: security_dest_bucket image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts - commands: @@ -4199,6 +4110,17 @@ steps: from_secret: static_asset_editions image: grafana/grafana-ci-deploy:1.3.3 name: publish-static-assets +- commands: + - ./bin/build artifacts storybook --tag ${DRONE_TAG} + depends_on: + - compile-build-cmd + environment: + GCP_KEY: + from_secret: gcp_key + PRERELEASE_BUCKET: + from_secret: prerelease_bucket + image: grafana/grafana-ci-deploy:1.3.3 + name: publish-storybook trigger: event: - promote @@ -4234,18 +4156,14 @@ steps: image: golang:1.20.4 name: compile-build-cmd - commands: - - ./bin/build artifacts packages --tag $${DRONE_TAG} --src-bucket $${PRERELEASE_BUCKET} + - ./bin/build artifacts packages --tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}} depends_on: - compile-build-cmd environment: - ENTERPRISE2_SECURITY_PREFIX: - from_secret: enterprise2_security_prefix GCP_KEY: from_secret: gcp_key PRERELEASE_BUCKET: from_secret: prerelease_bucket - SECURITY_DEST_BUCKET: - from_secret: security_dest_bucket image: grafana/grafana-ci-deploy:1.3.3 name: publish-artifacts - commands: @@ -4780,81 +4698,6 @@ volumes: path: //./pipe/docker_engine/ name: docker --- -clone: - disable: true -depends_on: [] -environment: - EDITION: enterprise -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: publish-artifacts-page -node: - type: no-parallel -platform: - arch: amd64 - os: linux -services: [] -steps: -- commands: - - mkdir -p bin - - curl -fL -o bin/grabpl https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v3.0.39/grabpl - - chmod +x bin/grabpl - image: byrnedo/alpine-curl:0.1.8 - name: grabpl -- commands: - - git clone "https://$${GITHUB_TOKEN}@github.com/grafana/grafana-enterprise.git" - - cd grafana-enterprise - - git checkout ${DRONE_TAG} - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: clone-enterprise -- commands: - - mv bin/grabpl /tmp/ - - rmdir bin - - mv grafana-enterprise /tmp/ - - /tmp/grabpl init-enterprise --github-token $${GITHUB_TOKEN} /tmp/grafana-enterprise - ${DRONE_TAG} - - mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json - - mkdir bin - - mv /tmp/grabpl bin/ - depends_on: - - clone-enterprise - - grabpl - environment: - GITHUB_TOKEN: - from_secret: github_token - image: grafana/build-container:1.7.4 - name: init-enterprise -- commands: - - go build -o ./bin/build -ldflags '-extldflags -static' ./pkg/build/cmd - depends_on: - - init-enterprise - environment: - CGO_ENABLED: 0 - image: golang:1.20.4 - name: compile-build-cmd -- commands: - - ./bin/build artifacts-page - depends_on: - - compile-build-cmd - environment: - GCP_KEY: - from_secret: gcp_key - image: grafana/build-container:1.7.4 - name: artifacts-page -trigger: - event: - - promote - target: security -type: docker -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- clone: retries: 3 depends_on: [] @@ -7291,36 +7134,18 @@ get: kind: secret name: aws_secret_access_key --- -get: - name: bucket - path: infra/data/ci/grafana-release-eng/security-bucket -kind: secret -name: security_dest_bucket ---- get: name: static_asset_editions path: infra/data/ci/grafana-release-eng/artifact-publishing kind: secret name: static_asset_editions --- -get: - name: security_prefix - path: infra/data/ci/grafana-release-eng/enterprise2 -kind: secret -name: enterprise2_security_prefix ---- get: name: cdn_path path: infra/data/ci/grafana-release-eng/enterprise2 kind: secret name: enterprise2-cdn-path --- -get: - name: security_prefix - path: infra/data/ci/grafana-release-eng/enterprise2 -kind: secret -name: enterprise2_security_prefix ---- get: name: gcp_service_account_base64 path: infra/data/ci/grafana-release-eng/rgm @@ -7364,6 +7189,6 @@ kind: secret name: delivery-bot-app-private-key --- kind: signature -hmac: 2cfe53f430c7cfec1b422a329b009b7c81cd52e897abb7a940115e524796178a +hmac: 5b576f21a7afb08759ad4ac7b74a54afcfbf9b5eac0e5894857ce4cdd0feed50 ... diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 87c1d911005..259c647ce96 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -4,7 +4,6 @@ This module returns all the pipelines used in the event of a release along with load( "scripts/drone/steps/lib.star", - "artifacts_page_step", "build_backend_step", "build_docker_images_step", "build_frontend_package_step", @@ -525,7 +524,6 @@ def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_tr publish_images_step( "enterprise2", "release", - mode = "enterprise2", docker_repo = "${{DOCKER_ENTERPRISE2_REPO}}", ), ], @@ -556,23 +554,16 @@ def enterprise2_pipelines(prefix = "", ver_mode = ver_mode, trigger = release_tr return pipelines -def publish_artifacts_step(mode): - security = "" - if mode == "security": - security = "--security " +def publish_artifacts_step(): return { "name": "publish-artifacts", "image": images["publish_image"], "environment": { "GCP_KEY": from_secret("gcp_key"), "PRERELEASE_BUCKET": from_secret("prerelease_bucket"), - "ENTERPRISE2_SECURITY_PREFIX": from_secret("enterprise2_security_prefix"), - "SECURITY_DEST_BUCKET": from_secret("security_dest_bucket"), }, "commands": [ - "./bin/build artifacts packages {}--tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}".format( - security, - ), + "./bin/build artifacts packages --tag $${{DRONE_TAG}} --src-bucket $${{PRERELEASE_BUCKET}}", ], "depends_on": ["compile-build-cmd"], } @@ -622,11 +613,10 @@ def publish_artifacts_pipelines(mode): } steps = [ compile_build_cmd(), - publish_artifacts_step(mode), + publish_artifacts_step(), publish_static_assets_step(), + publish_storybook_step(), ] - if mode != "security": - steps.extend([publish_storybook_step()]) return [ pipeline( @@ -709,27 +699,6 @@ def publish_npm_pipelines(): ), ] -def artifacts_page_pipeline(): - trigger = { - "event": ["promote"], - "target": "security", - } - return [ - pipeline( - name = "publish-artifacts-page", - trigger = trigger, - steps = [ - download_grabpl_step(), - clone_enterprise_step(source = "${DRONE_TAG}"), - init_enterprise_step("release"), - compile_build_cmd("enterprise"), - artifacts_page_step(), - ], - edition = "enterprise", - environment = {"EDITION": "enterprise"}, - ), - ] - def integration_test_pipelines(): """ Trigger integration tests on release builds diff --git a/scripts/drone/pipelines/build.star b/scripts/drone/pipelines/build.star index f64af736875..6add0e398ac 100644 --- a/scripts/drone/pipelines/build.star +++ b/scripts/drone/pipelines/build.star @@ -112,14 +112,12 @@ def build_e2e(trigger, ver_mode): publish_images_step( docker_repo = "grafana", edition = edition, - mode = "", trigger = trigger_oss, ver_mode = ver_mode, ), publish_images_step( docker_repo = "grafana-oss", edition = edition, - mode = "", trigger = trigger_oss, ver_mode = ver_mode, ), @@ -155,7 +153,6 @@ def build_e2e(trigger, ver_mode): publish_images_step( docker_repo = "grafana", edition = edition, - mode = "", trigger = trigger_oss, ver_mode = ver_mode, ), diff --git a/scripts/drone/pipelines/publish_images.star b/scripts/drone/pipelines/publish_images.star index 18c94dcbca3..a644e13dc02 100644 --- a/scripts/drone/pipelines/publish_images.star +++ b/scripts/drone/pipelines/publish_images.star @@ -15,14 +15,13 @@ load( "pipeline", ) -def publish_image_steps(edition, mode, docker_repo): +def publish_image_steps(edition, docker_repo): """Generates the steps used for publising Docker images using grabpl. Args: edition: controls which version of an image is fetched in the case of a release. It also controls which publishing implementation is used. If edition == 'oss', it additionally publishes the grafana/grafana-oss repository. - mode: uses to control the publishing of security images when mode == 'security'. docker_repo: the Docker image name. It is combined with the 'grafana/' library prefix. @@ -34,12 +33,12 @@ def publish_image_steps(edition, mode, docker_repo): download_grabpl_step(), compile_build_cmd(), fetch_images_step(edition), - publish_images_step(edition, "release", mode, docker_repo), + publish_images_step(edition, "release", docker_repo), ] if edition == "oss": steps.append( - publish_images_step(edition, "release", mode, "grafana-oss"), + publish_images_step(edition, "release", "grafana-oss"), ) return steps @@ -59,7 +58,7 @@ def publish_image_pipelines_public(): pipeline( name = "publish-docker-oss-{}".format(mode), trigger = trigger, - steps = publish_image_steps(edition = "oss", mode = mode, docker_repo = "grafana"), + steps = publish_image_steps(edition = "oss", docker_repo = "grafana"), edition = "", environment = {"EDITION": "oss"}, ), @@ -68,27 +67,6 @@ def publish_image_pipelines_public(): trigger = trigger, steps = publish_image_steps( edition = "enterprise", - mode = mode, - docker_repo = "grafana-enterprise", - ), - edition = "", - environment = {"EDITION": "enterprise"}, - ), - ] - -def publish_image_pipelines_security(): - mode = "security" - trigger = { - "event": ["promote"], - "target": [mode], - } - return [ - pipeline( - name = "publish-docker-enterprise-{}".format(mode), - trigger = trigger, - steps = publish_image_steps( - edition = "enterprise", - mode = mode, docker_repo = "grafana-enterprise", ), edition = "", diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index ceb8cc2725d..906bd7c6a77 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -1130,7 +1130,7 @@ def fetch_images_step(edition): "volumes": [{"name": "docker", "path": "/var/run/docker.sock"}], } -def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None): +def publish_images_step(edition, ver_mode, docker_repo, trigger = None): """Generates a step for publishing public Docker images with grabpl. Args: @@ -1138,7 +1138,6 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None): It also controls which publishing implementation is used. ver_mode: controls whether the image needs to be built or retrieved from a previous build. If ver_mode == 'release', the previously built image is fetched instead of being built again. - mode: uses to control the publishing of security images when mode == 'security'. docker_repo: the Docker image name. It is combined with the 'grafana/' library prefix. trigger: a Drone trigger for the pipeline. @@ -1149,10 +1148,6 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None): """ name = docker_repo docker_repo = "grafana/{}".format(docker_repo) - if mode == "security": - mode = "--{} ".format(mode) - else: - mode = "" environment = { "GCP_KEY": from_secret("gcp_key"), @@ -1163,8 +1158,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger = None): "GITHUB_APP_PRIVATE_KEY": from_secret("delivery-bot-app-private-key"), } - cmd = "./bin/grabpl artifacts docker publish {}--dockerhub-repo {}".format( - mode, + cmd = "./bin/grabpl artifacts docker publish --dockerhub-repo {}".format( docker_repo, ) @@ -1654,21 +1648,6 @@ def trigger_test_release(): }, } -def artifacts_page_step(): - return { - "name": "artifacts-page", - "image": images["build_image"], - "depends_on": [ - "compile-build-cmd", - ], - "environment": { - "GCP_KEY": from_secret("gcp_key"), - }, - "commands": [ - "./bin/build artifacts-page", - ], - } - def end_to_end_tests_deps(): return [ "end-to-end-tests-dashboards-suite", diff --git a/scripts/drone/vault.star b/scripts/drone/vault.star index cbddf044f3f..9b70e094a4f 100644 --- a/scripts/drone/vault.star +++ b/scripts/drone/vault.star @@ -99,31 +99,16 @@ def secrets(): "secret/data/common/aws-marketplace", "aws_secret_access_key", ), - vault_secret( - "security_dest_bucket", - "infra/data/ci/grafana-release-eng/security-bucket", - "bucket", - ), vault_secret( "static_asset_editions", "infra/data/ci/grafana-release-eng/artifact-publishing", "static_asset_editions", ), - vault_secret( - "enterprise2_security_prefix", - "infra/data/ci/grafana-release-eng/enterprise2", - "security_prefix", - ), vault_secret( "enterprise2-cdn-path", "infra/data/ci/grafana-release-eng/enterprise2", "cdn_path", ), - vault_secret( - "enterprise2_security_prefix", - "infra/data/ci/grafana-release-eng/enterprise2", - "security_prefix", - ), vault_secret( rgm_gcp_key_base64, "infra/data/ci/grafana-release-eng/rgm",