CI: Backport CI/Release related code to v9.3.x (#62752)
* Batch-move everything
* go mod tidy
* make drone
* Remove genversions
* Bump alpine image
* Revert back pkg/build/docker/build.go
* Make sure correct enterprise branch is checked out
* Add enterprise2 version
* Remove extensions
* Bump build container
* backport node 18 test fix
(cherry picked from commit 4ff03fdbfb)
* Update scripts/drone
* Add more commands
* Fix starlark link
* Copy .drone.star
* Add drone target branch for custom events
---------
This commit is contained in:
Dimitris Sotirakis
@@ -1,99 +1,115 @@
|
||||
load('scripts/drone/vault.star', 'from_secret', 'pull_secret')
|
||||
load('scripts/drone/steps/lib.star', 'publish_image', 'compile_build_cmd')
|
||||
"""
|
||||
This module provides functions for cronjob pipelines and steps used within.
|
||||
"""
|
||||
|
||||
aquasec_trivy_image = 'aquasec/trivy:0.21.0'
|
||||
load("scripts/drone/vault.star", "from_secret")
|
||||
load(
|
||||
"scripts/drone/steps/lib.star",
|
||||
"compile_build_cmd",
|
||||
"publish_image",
|
||||
)
|
||||
|
||||
def cronjobs(edition):
|
||||
grafana_com_nightly_pipeline = cron_job_pipeline(
|
||||
cronName='grafana-com-nightly',
|
||||
name='grafana-com-nightly',
|
||||
steps=[compile_build_cmd(),post_to_grafana_com_step()]
|
||||
)
|
||||
aquasec_trivy_image = "aquasec/trivy:0.21.0"
|
||||
|
||||
def cronjobs():
|
||||
return [
|
||||
scan_docker_image_pipeline(edition, 'latest'),
|
||||
scan_docker_image_pipeline(edition, 'main'),
|
||||
scan_docker_image_pipeline(edition, 'latest-ubuntu'),
|
||||
scan_docker_image_pipeline(edition, 'main-ubuntu'),
|
||||
grafana_com_nightly_pipeline,
|
||||
scan_docker_image_pipeline("latest"),
|
||||
scan_docker_image_pipeline("main"),
|
||||
scan_docker_image_pipeline("latest-ubuntu"),
|
||||
scan_docker_image_pipeline("main-ubuntu"),
|
||||
grafana_com_nightly_pipeline(),
|
||||
]
|
||||
|
||||
def cron_job_pipeline(cronName, name, steps):
|
||||
return {
|
||||
'kind': 'pipeline',
|
||||
'type': 'docker',
|
||||
'platform': {
|
||||
'os': 'linux',
|
||||
'arch': 'amd64',
|
||||
"kind": "pipeline",
|
||||
"type": "docker",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "amd64",
|
||||
},
|
||||
'name': name,
|
||||
'trigger': {
|
||||
'event': 'cron',
|
||||
'cron': cronName,
|
||||
"name": name,
|
||||
"trigger": {
|
||||
"event": "cron",
|
||||
"cron": cronName,
|
||||
},
|
||||
'clone': {
|
||||
'retries': 3,
|
||||
"clone": {
|
||||
"retries": 3,
|
||||
},
|
||||
'steps': steps,
|
||||
"steps": steps,
|
||||
}
|
||||
|
||||
def scan_docker_image_pipeline(edition, tag):
|
||||
if edition != 'oss':
|
||||
edition='grafana-enterprise'
|
||||
else:
|
||||
edition='grafana'
|
||||
def scan_docker_image_pipeline(tag):
|
||||
"""Generates a cronjob pipeline for nightly scans of grafana Docker images.
|
||||
|
||||
dockerImage='grafana/{}:{}'.format(edition, tag)
|
||||
Args:
|
||||
tag: determines which image tag is scanned.
|
||||
|
||||
Returns:
|
||||
Drone cronjob pipeline.
|
||||
"""
|
||||
docker_image = "grafana/grafana:{}".format(tag)
|
||||
|
||||
return cron_job_pipeline(
|
||||
cronName='nightly',
|
||||
name='scan-' + dockerImage + '-image',
|
||||
steps=[
|
||||
scan_docker_image_unkown_low_medium_vulnerabilities_step(dockerImage),
|
||||
scan_docker_image_high_critical_vulnerabilities_step(dockerImage),
|
||||
slack_job_failed_step('grafana-backend-ops', dockerImage),
|
||||
])
|
||||
cronName = "nightly",
|
||||
name = "scan-" + docker_image + "-image",
|
||||
steps = [
|
||||
scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image),
|
||||
scan_docker_image_high_critical_vulnerabilities_step(docker_image),
|
||||
slack_job_failed_step("grafana-backend-ops", docker_image),
|
||||
],
|
||||
)
|
||||
|
||||
def scan_docker_image_unkown_low_medium_vulnerabilities_step(dockerImage):
|
||||
def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):
|
||||
return {
|
||||
'name': 'scan-unkown-low-medium-vulnerabilities',
|
||||
'image': aquasec_trivy_image,
|
||||
'commands': [
|
||||
'trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ' + dockerImage,
|
||||
"name": "scan-unkown-low-medium-vulnerabilities",
|
||||
"image": aquasec_trivy_image,
|
||||
"commands": [
|
||||
"trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image,
|
||||
],
|
||||
}
|
||||
|
||||
def scan_docker_image_high_critical_vulnerabilities_step(dockerImage):
|
||||
def scan_docker_image_high_critical_vulnerabilities_step(docker_image):
|
||||
return {
|
||||
'name': 'scan-high-critical-vulnerabilities',
|
||||
'image': aquasec_trivy_image,
|
||||
'commands': [
|
||||
'trivy --exit-code 1 --severity HIGH,CRITICAL ' + dockerImage,
|
||||
"name": "scan-high-critical-vulnerabilities",
|
||||
"image": aquasec_trivy_image,
|
||||
"commands": [
|
||||
"trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image,
|
||||
],
|
||||
}
|
||||
|
||||
def slack_job_failed_step(channel, image):
|
||||
return {
|
||||
'name': 'slack-notify-failure',
|
||||
'image': 'plugins/slack',
|
||||
'settings': {
|
||||
'webhook': from_secret('slack_webhook_backend'),
|
||||
'channel': channel,
|
||||
'template': 'Nightly docker image scan job for ' + image + ' failed: {{build.link}}',
|
||||
"name": "slack-notify-failure",
|
||||
"image": "plugins/slack",
|
||||
"settings": {
|
||||
"webhook": from_secret("slack_webhook_backend"),
|
||||
"channel": channel,
|
||||
"template": "Nightly docker image scan job for " +
|
||||
image +
|
||||
" failed: {{build.link}}",
|
||||
},
|
||||
'when': {
|
||||
'status': 'failure'
|
||||
}
|
||||
"when": {"status": "failure"},
|
||||
}
|
||||
|
||||
def post_to_grafana_com_step():
|
||||
return {
|
||||
'name': 'post-to-grafana-com',
|
||||
'image': publish_image,
|
||||
'environment': {
|
||||
'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
|
||||
'GCP_KEY': from_secret('gcp_key'),
|
||||
},
|
||||
'depends_on': ['compile-build-cmd'],
|
||||
'commands': ['./bin/build publish grafana-com --edition oss'],
|
||||
}
|
||||
"name": "post-to-grafana-com",
|
||||
"image": publish_image,
|
||||
"environment": {
|
||||
"GRAFANA_COM_API_KEY": from_secret("grafana_api_key"),
|
||||
"GCP_KEY": from_secret("gcp_key"),
|
||||
},
|
||||
"depends_on": ["compile-build-cmd"],
|
||||
"commands": ["./bin/build publish grafana-com --edition oss"],
|
||||
}
|
||||
|
||||
def grafana_com_nightly_pipeline():
|
||||
return cron_job_pipeline(
|
||||
cronName = "grafana-com-nightly",
|
||||
name = "grafana-com-nightly",
|
||||
steps = [
|
||||
compile_build_cmd(),
|
||||
post_to_grafana_com_step(),
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1,108 +1,116 @@
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
'notify_pipeline',
|
||||
'failure_template',
|
||||
'drone_change_template',
|
||||
)
|
||||
"""
|
||||
This module returns all the pipelines used in the event of pushes to the main branch.
|
||||
"""
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/docs.star',
|
||||
'docs_pipelines',
|
||||
'trigger_docs_main',
|
||||
"scripts/drone/utils/utils.star",
|
||||
"drone_change_template",
|
||||
"failure_template",
|
||||
"notify_pipeline",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/test_frontend.star',
|
||||
'test_frontend',
|
||||
"scripts/drone/pipelines/docs.star",
|
||||
"docs_pipelines",
|
||||
"trigger_docs_main",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/test_backend.star',
|
||||
'test_backend',
|
||||
"scripts/drone/pipelines/test_frontend.star",
|
||||
"test_frontend",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/integration_tests.star',
|
||||
'integration_tests',
|
||||
"scripts/drone/pipelines/test_backend.star",
|
||||
"test_backend",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/build.star',
|
||||
'build_e2e',
|
||||
"scripts/drone/pipelines/integration_tests.star",
|
||||
"integration_tests",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/windows.star',
|
||||
'windows',
|
||||
"scripts/drone/pipelines/build.star",
|
||||
"build_e2e",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/trigger_downstream.star',
|
||||
'enterprise_downstream_pipeline',
|
||||
"scripts/drone/pipelines/windows.star",
|
||||
"windows",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/lint_backend.star',
|
||||
'lint_backend_pipeline',
|
||||
"scripts/drone/pipelines/trigger_downstream.star",
|
||||
"enterprise_downstream_pipeline",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/lint_frontend.star',
|
||||
'lint_frontend_pipeline',
|
||||
"scripts/drone/pipelines/lint_backend.star",
|
||||
"lint_backend_pipeline",
|
||||
)
|
||||
load(
|
||||
"scripts/drone/pipelines/lint_frontend.star",
|
||||
"lint_frontend_pipeline",
|
||||
)
|
||||
|
||||
load('scripts/drone/vault.star', 'from_secret')
|
||||
|
||||
|
||||
ver_mode = 'main'
|
||||
ver_mode = "main"
|
||||
trigger = {
|
||||
'event': ['push',],
|
||||
'branch': 'main',
|
||||
'paths': {
|
||||
'exclude': [
|
||||
'*.md',
|
||||
'docs/**',
|
||||
'latest.json',
|
||||
"event": [
|
||||
"push",
|
||||
],
|
||||
"branch": "main",
|
||||
"paths": {
|
||||
"exclude": [
|
||||
"*.md",
|
||||
"docs/**",
|
||||
"latest.json",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
def main_pipelines(edition):
|
||||
def main_pipelines():
|
||||
drone_change_trigger = {
|
||||
'event': ['push',],
|
||||
'branch': 'main',
|
||||
'repo': [
|
||||
'grafana/grafana',
|
||||
"event": [
|
||||
"push",
|
||||
],
|
||||
'paths': {
|
||||
'include': [
|
||||
'.drone.yml',
|
||||
"branch": "main",
|
||||
"repo": [
|
||||
"grafana/grafana",
|
||||
],
|
||||
"paths": {
|
||||
"include": [
|
||||
".drone.yml",
|
||||
],
|
||||
'exclude': [
|
||||
'exclude',
|
||||
"exclude": [
|
||||
"exclude",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
pipelines = [
|
||||
docs_pipelines(edition, ver_mode, trigger_docs_main()),
|
||||
docs_pipelines(ver_mode, trigger_docs_main()),
|
||||
test_frontend(trigger, ver_mode),
|
||||
lint_frontend_pipeline(trigger, ver_mode),
|
||||
test_backend(trigger, ver_mode),
|
||||
lint_backend_pipeline(trigger, ver_mode),
|
||||
build_e2e(trigger, ver_mode, edition),
|
||||
integration_tests(trigger, ver_mode, edition),
|
||||
windows(trigger, edition, ver_mode),
|
||||
notify_pipeline(
|
||||
name='notify-drone-changes', slack_channel='slack-webhooks-test', trigger=drone_change_trigger,
|
||||
template=drone_change_template, secret='drone-changes-webhook',
|
||||
),
|
||||
enterprise_downstream_pipeline(edition, ver_mode),
|
||||
notify_pipeline(
|
||||
name='main-notify', slack_channel='grafana-ci-notifications', trigger=dict(trigger, status=['failure']),
|
||||
depends_on=['main-test-frontend', 'main-test-backend', 'main-build-e2e-publish', 'main-integration-tests', 'main-windows'],
|
||||
template=failure_template, secret='slack_webhook'
|
||||
)]
|
||||
build_e2e(trigger, ver_mode),
|
||||
integration_tests(trigger, prefix = ver_mode, ver_mode = ver_mode),
|
||||
windows(trigger, edition = "oss", ver_mode = ver_mode),
|
||||
notify_pipeline(
|
||||
name = "notify-drone-changes",
|
||||
slack_channel = "slack-webhooks-test",
|
||||
trigger = drone_change_trigger,
|
||||
template = drone_change_template,
|
||||
secret = "drone-changes-webhook",
|
||||
),
|
||||
enterprise_downstream_pipeline(),
|
||||
notify_pipeline(
|
||||
name = "main-notify",
|
||||
slack_channel = "grafana-ci-notifications",
|
||||
trigger = dict(trigger, status = ["failure"]),
|
||||
depends_on = [
|
||||
"main-test-frontend",
|
||||
"main-test-backend",
|
||||
"main-build-e2e-publish",
|
||||
"main-integration-tests",
|
||||
"main-windows",
|
||||
],
|
||||
template = failure_template,
|
||||
secret = "slack_webhook",
|
||||
),
|
||||
]
|
||||
|
||||
return pipelines
|
||||
|
||||
+126
-57
@@ -1,85 +1,155 @@
|
||||
load(
|
||||
'scripts/drone/utils/utils.star',
|
||||
'pipeline',
|
||||
)
|
||||
"""
|
||||
This module returns all pipelines used in the event of a pull request.
|
||||
It also includes a function generating a PR trigger from a list of included and excluded paths.
|
||||
"""
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/test_frontend.star',
|
||||
'test_frontend',
|
||||
"scripts/drone/pipelines/test_frontend.star",
|
||||
"test_frontend",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/test_backend.star',
|
||||
'test_backend',
|
||||
"scripts/drone/pipelines/test_backend.star",
|
||||
"test_backend",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/integration_tests.star',
|
||||
'integration_tests',
|
||||
"scripts/drone/pipelines/integration_tests.star",
|
||||
"integration_tests",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/build.star',
|
||||
'build_e2e',
|
||||
"scripts/drone/pipelines/build.star",
|
||||
"build_e2e",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/verify_drone.star',
|
||||
'verify_drone',
|
||||
"scripts/drone/pipelines/verify_drone.star",
|
||||
"verify_drone",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/docs.star',
|
||||
'docs_pipelines',
|
||||
'trigger_docs_pr',
|
||||
"scripts/drone/pipelines/verify_starlark.star",
|
||||
"verify_starlark",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/shellcheck.star',
|
||||
'shellcheck_pipeline',
|
||||
"scripts/drone/pipelines/docs.star",
|
||||
"docs_pipelines",
|
||||
"trigger_docs_pr",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/lint_backend.star',
|
||||
'lint_backend_pipeline',
|
||||
"scripts/drone/pipelines/shellcheck.star",
|
||||
"shellcheck_pipeline",
|
||||
)
|
||||
|
||||
load(
|
||||
'scripts/drone/pipelines/lint_frontend.star',
|
||||
'lint_frontend_pipeline',
|
||||
"scripts/drone/pipelines/lint_backend.star",
|
||||
"lint_backend_pipeline",
|
||||
)
|
||||
load(
|
||||
"scripts/drone/pipelines/lint_frontend.star",
|
||||
"lint_frontend_pipeline",
|
||||
)
|
||||
|
||||
ver_mode = 'pr'
|
||||
ver_mode = "pr"
|
||||
trigger = {
|
||||
'event': [
|
||||
'pull_request',
|
||||
"event": [
|
||||
"pull_request",
|
||||
],
|
||||
'paths': {
|
||||
'exclude': [
|
||||
'*.md',
|
||||
'docs/**',
|
||||
'latest.json',
|
||||
"paths": {
|
||||
"exclude": [
|
||||
"*.md",
|
||||
"docs/**",
|
||||
"latest.json",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def pr_pipelines(edition):
|
||||
def pr_pipelines():
|
||||
return [
|
||||
verify_drone(get_pr_trigger(include_paths=['scripts/drone/**', '.drone.yml', '.drone.star']), ver_mode),
|
||||
test_frontend(get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), ver_mode),
|
||||
lint_frontend_pipeline(get_pr_trigger(exclude_paths=['pkg/**', 'packaging/**', 'go.sum', 'go.mod']), ver_mode),
|
||||
test_backend(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json', 'devenv/**']), ver_mode),
|
||||
lint_backend_pipeline(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json', 'devenv/**']), ver_mode),
|
||||
build_e2e(trigger, ver_mode, edition),
|
||||
integration_tests(get_pr_trigger(include_paths=['pkg/**', 'packaging/**', '.drone.yml', 'conf/**', 'go.sum', 'go.mod', 'public/app/plugins/**/plugin.json']), ver_mode, edition),
|
||||
docs_pipelines(edition, ver_mode, trigger_docs_pr()),
|
||||
verify_drone(
|
||||
get_pr_trigger(
|
||||
include_paths = ["scripts/drone/**", ".drone.yml", ".drone.star"],
|
||||
),
|
||||
ver_mode,
|
||||
),
|
||||
verify_starlark(
|
||||
get_pr_trigger(
|
||||
include_paths = ["scripts/drone/**", ".drone.star"],
|
||||
),
|
||||
ver_mode,
|
||||
),
|
||||
test_frontend(
|
||||
get_pr_trigger(
|
||||
exclude_paths = ["pkg/**", "packaging/**", "go.sum", "go.mod"],
|
||||
),
|
||||
ver_mode,
|
||||
),
|
||||
lint_frontend_pipeline(
|
||||
get_pr_trigger(
|
||||
exclude_paths = ["pkg/**", "packaging/**", "go.sum", "go.mod"],
|
||||
),
|
||||
ver_mode,
|
||||
),
|
||||
test_backend(
|
||||
get_pr_trigger(
|
||||
include_paths = [
|
||||
"pkg/**",
|
||||
"packaging/**",
|
||||
".drone.yml",
|
||||
"conf/**",
|
||||
"go.sum",
|
||||
"go.mod",
|
||||
"public/app/plugins/**/plugin.json",
|
||||
"devenv/**",
|
||||
],
|
||||
),
|
||||
ver_mode,
|
||||
),
|
||||
lint_backend_pipeline(
|
||||
get_pr_trigger(
|
||||
include_paths = [
|
||||
"pkg/**",
|
||||
"packaging/**",
|
||||
"conf/**",
|
||||
"go.sum",
|
||||
"go.mod",
|
||||
"public/app/plugins/**/plugin.json",
|
||||
"devenv/**",
|
||||
".bingo/**",
|
||||
],
|
||||
),
|
||||
ver_mode,
|
||||
),
|
||||
build_e2e(trigger, ver_mode),
|
||||
integration_tests(
|
||||
get_pr_trigger(
|
||||
include_paths = [
|
||||
"pkg/**",
|
||||
"packaging/**",
|
||||
".drone.yml",
|
||||
"conf/**",
|
||||
"go.sum",
|
||||
"go.mod",
|
||||
"public/app/plugins/**/plugin.json",
|
||||
],
|
||||
),
|
||||
prefix = ver_mode,
|
||||
),
|
||||
docs_pipelines(ver_mode, trigger_docs_pr()),
|
||||
shellcheck_pipeline(),
|
||||
]
|
||||
|
||||
def get_pr_trigger(include_paths = None, exclude_paths = None):
|
||||
"""Generates a trigger filter from the lists of included and excluded path patterns.
|
||||
|
||||
def get_pr_trigger(include_paths=None, exclude_paths=None):
|
||||
paths_ex = ['docs/**', '*.md']
|
||||
This function is primarily intended to generate a trigger for code changes
|
||||
as the patterns 'docs/**' and '*.md' are always excluded.
|
||||
|
||||
Args:
|
||||
include_paths: a list of path patterns using the same syntax as gitignore.
|
||||
Changes affecting files matching these path patterns trigger the pipeline.
|
||||
exclude_paths: a list of path patterns using the same syntax as gitignore.
|
||||
Changes affecting files matching these path patterns do not trigger the pipeline.
|
||||
|
||||
Returns:
|
||||
Drone trigger.
|
||||
"""
|
||||
paths_ex = ["docs/**", "*.md"]
|
||||
paths_in = []
|
||||
if include_paths:
|
||||
for path in include_paths:
|
||||
@@ -88,12 +158,11 @@ def get_pr_trigger(include_paths=None, exclude_paths=None):
|
||||
for path in exclude_paths:
|
||||
paths_ex.extend([path])
|
||||
return {
|
||||
'event': [
|
||||
'pull_request',
|
||||
"event": [
|
||||
"pull_request",
|
||||
],
|
||||
'paths': {
|
||||
'exclude': paths_ex,
|
||||
'include': paths_in,
|
||||
"paths": {
|
||||
"exclude": paths_ex,
|
||||
"include": paths_in,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
+547
-344
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user