CI: Lint starlark files with buildifier (#59157)
* Add verify-starlark build action that returns an error for starlark files with lint Relies on `buildifier` tool. Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add verify_starlark_step to PR pipeline Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Manually fetch buildifier in curl_image until a new build_image is created Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Format with buildifier Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Remove all unused variables retaining one unused function Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use snake_case for variable Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Replace deprecated dictionary concatenation with .update() method Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Start adding docstrings for all modules and functions Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Prefer os.WriteFile as ioutil.WriteFile has been deprecated since go 1.16 Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Attempt to document the behavior of the init_enterprise_step Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document test_backend pipeline Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document enterprise_downstream_step Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document the pipeline utility function Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document publish_images_step Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document publish_images_steps Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document enterprise2_pipelines function Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add tags table for Starlark files. Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document test_frontend Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document windows function Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add docstrings to verifystarlark functions Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Refactor error handling to be more clear and document complex behavior Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Split errors into execution errors and verification errors Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document all other library functions Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add local variables to TAGS Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Add blank line between all Args and Returns sections Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Fix new linting errors Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Lint new Starlark files Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Correct buildifier binary mv Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Document the need to set nofile ulimit to at least 2048 Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Update build-container to include buildifier Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Ensure buildifier binary is executable Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Fix valid content test Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Simply return execution error Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Only check files rather than fixing them Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use updated build-container with executable buildifier Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Test that context cancellation stops execution Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Simplify error handling Return execution errors that short circuit WalkDir rather than separately tracking that error. Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Remove fetching of buildifier binary now that it is in the build-container Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use build image in verify-starlark step Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use semver tag The image is the same but uses a semver tag to make it clearer that this is a forward upgrade from the old version. Signed-off-by: Jack Baldry <jack.baldry@grafana.com> * Use node 18 image with buildifier Signed-off-by: Jack Baldry <jack.baldry@grafana.com> --------- Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
This commit is contained in:
Jack Baldry
GitHub
@@ -1,110 +1,114 @@
|
||||
load('scripts/drone/vault.star', 'from_secret')
|
||||
"""
|
||||
This module provides functions for cronjob pipelines and steps used within.
|
||||
"""
|
||||
|
||||
load("scripts/drone/vault.star", "from_secret")
|
||||
load(
|
||||
'scripts/drone/steps/lib.star',
|
||||
'publish_image',
|
||||
'compile_build_cmd',
|
||||
"scripts/drone/steps/lib.star",
|
||||
"compile_build_cmd",
|
||||
"publish_image",
|
||||
)
|
||||
|
||||
aquasec_trivy_image = 'aquasec/trivy:0.21.0'
|
||||
|
||||
aquasec_trivy_image = "aquasec/trivy:0.21.0"
|
||||
|
||||
def cronjobs():
|
||||
return [
|
||||
scan_docker_image_pipeline('latest'),
|
||||
scan_docker_image_pipeline('main'),
|
||||
scan_docker_image_pipeline('latest-ubuntu'),
|
||||
scan_docker_image_pipeline('main-ubuntu'),
|
||||
scan_docker_image_pipeline("latest"),
|
||||
scan_docker_image_pipeline("main"),
|
||||
scan_docker_image_pipeline("latest-ubuntu"),
|
||||
scan_docker_image_pipeline("main-ubuntu"),
|
||||
grafana_com_nightly_pipeline(),
|
||||
]
|
||||
|
||||
|
||||
def cron_job_pipeline(cronName, name, steps):
|
||||
return {
|
||||
'kind': 'pipeline',
|
||||
'type': 'docker',
|
||||
'platform': {
|
||||
'os': 'linux',
|
||||
'arch': 'amd64',
|
||||
"kind": "pipeline",
|
||||
"type": "docker",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "amd64",
|
||||
},
|
||||
'name': name,
|
||||
'trigger': {
|
||||
'event': 'cron',
|
||||
'cron': cronName,
|
||||
"name": name,
|
||||
"trigger": {
|
||||
"event": "cron",
|
||||
"cron": cronName,
|
||||
},
|
||||
'clone': {
|
||||
'retries': 3,
|
||||
"clone": {
|
||||
"retries": 3,
|
||||
},
|
||||
'steps': steps,
|
||||
"steps": steps,
|
||||
}
|
||||
|
||||
|
||||
def scan_docker_image_pipeline(tag):
|
||||
dockerImage = 'grafana/{}:{}'.format('grafana', tag)
|
||||
"""Generates a cronjob pipeline for nightly scans of grafana Docker images.
|
||||
|
||||
Args:
|
||||
tag: determines which image tag is scanned.
|
||||
|
||||
Returns:
|
||||
Drone cronjob pipeline.
|
||||
"""
|
||||
docker_image = "grafana/grafana:{}".format(tag)
|
||||
|
||||
return cron_job_pipeline(
|
||||
cronName='nightly',
|
||||
name='scan-' + dockerImage + '-image',
|
||||
steps=[
|
||||
scan_docker_image_unkown_low_medium_vulnerabilities_step(dockerImage),
|
||||
scan_docker_image_high_critical_vulnerabilities_step(dockerImage),
|
||||
slack_job_failed_step('grafana-backend-ops', dockerImage),
|
||||
cronName = "nightly",
|
||||
name = "scan-" + docker_image + "-image",
|
||||
steps = [
|
||||
scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image),
|
||||
scan_docker_image_high_critical_vulnerabilities_step(docker_image),
|
||||
slack_job_failed_step("grafana-backend-ops", docker_image),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def scan_docker_image_unkown_low_medium_vulnerabilities_step(dockerImage):
|
||||
def scan_docker_image_unkown_low_medium_vulnerabilities_step(docker_image):
|
||||
return {
|
||||
'name': 'scan-unkown-low-medium-vulnerabilities',
|
||||
'image': aquasec_trivy_image,
|
||||
'commands': [
|
||||
'trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM ' + dockerImage,
|
||||
"name": "scan-unkown-low-medium-vulnerabilities",
|
||||
"image": aquasec_trivy_image,
|
||||
"commands": [
|
||||
"trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM " + docker_image,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def scan_docker_image_high_critical_vulnerabilities_step(dockerImage):
|
||||
def scan_docker_image_high_critical_vulnerabilities_step(docker_image):
|
||||
return {
|
||||
'name': 'scan-high-critical-vulnerabilities',
|
||||
'image': aquasec_trivy_image,
|
||||
'commands': [
|
||||
'trivy --exit-code 1 --severity HIGH,CRITICAL ' + dockerImage,
|
||||
"name": "scan-high-critical-vulnerabilities",
|
||||
"image": aquasec_trivy_image,
|
||||
"commands": [
|
||||
"trivy --exit-code 1 --severity HIGH,CRITICAL " + docker_image,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def slack_job_failed_step(channel, image):
|
||||
return {
|
||||
'name': 'slack-notify-failure',
|
||||
'image': 'plugins/slack',
|
||||
'settings': {
|
||||
'webhook': from_secret('slack_webhook_backend'),
|
||||
'channel': channel,
|
||||
'template': 'Nightly docker image scan job for '
|
||||
+ image
|
||||
+ ' failed: {{build.link}}',
|
||||
"name": "slack-notify-failure",
|
||||
"image": "plugins/slack",
|
||||
"settings": {
|
||||
"webhook": from_secret("slack_webhook_backend"),
|
||||
"channel": channel,
|
||||
"template": "Nightly docker image scan job for " +
|
||||
image +
|
||||
" failed: {{build.link}}",
|
||||
},
|
||||
'when': {'status': 'failure'},
|
||||
"when": {"status": "failure"},
|
||||
}
|
||||
|
||||
|
||||
def post_to_grafana_com_step():
|
||||
return {
|
||||
'name': 'post-to-grafana-com',
|
||||
'image': publish_image,
|
||||
'environment': {
|
||||
'GRAFANA_COM_API_KEY': from_secret('grafana_api_key'),
|
||||
'GCP_KEY': from_secret('gcp_key'),
|
||||
"name": "post-to-grafana-com",
|
||||
"image": publish_image,
|
||||
"environment": {
|
||||
"GRAFANA_COM_API_KEY": from_secret("grafana_api_key"),
|
||||
"GCP_KEY": from_secret("gcp_key"),
|
||||
},
|
||||
'depends_on': ['compile-build-cmd'],
|
||||
'commands': ['./bin/build publish grafana-com --edition oss'],
|
||||
"depends_on": ["compile-build-cmd"],
|
||||
"commands": ["./bin/build publish grafana-com --edition oss"],
|
||||
}
|
||||
|
||||
|
||||
def grafana_com_nightly_pipeline():
|
||||
return cron_job_pipeline(
|
||||
cronName='grafana-com-nightly',
|
||||
name='grafana-com-nightly',
|
||||
steps=[
|
||||
cronName = "grafana-com-nightly",
|
||||
name = "grafana-com-nightly",
|
||||
steps = [
|
||||
compile_build_cmd(),
|
||||
post_to_grafana_com_step(),
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user