Chore: Rename scripts/star dir to scripts/drone (#38910)

* Rename scripts/star dir to scripts/drone

* Update CODEOWNERS after paths change
This commit is contained in:
Dimitris Sotirakis
2021-09-07 12:15:04 +03:00
committed by GitHub
parent 058ac6becf
commit 5f10843203
11 changed files with 21 additions and 21 deletions
+69
View File
@@ -0,0 +1,69 @@
load('scripts/drone/vault.star', 'from_secret')
def cronjobs(edition):
if edition != 'oss':
edition='grafana-enterprise'
else:
edition='grafana'
trigger = {
'event': 'cron',
'cron': 'nightly',
}
platform_conf = {
'os': 'linux',
'arch': 'amd64',
}
steps=[
scan_docker_image_unkown_low_medium_vulnerabilities_step(edition),
scan_docker_image_high_critical_vulnerabilities_step(edition),
slack_job_failed_step('grafana-backend-ops'),
]
return [
{
'kind': 'pipeline',
'type': 'docker',
'platform': platform_conf,
'name': 'scan-docker-images',
'trigger': trigger,
'services': [],
'steps': steps,
}
]
def scan_docker_image_unkown_low_medium_vulnerabilities_step(edition):
tags=['latest', 'main', 'latest-ubuntu', 'main-ubuntu']
commands=[]
for t in tags:
commands.append('trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/{}:{}'.format(edition,t))
return {
'name': 'scan-docker-images-unkown-low-medium-vulnerabilities',
'image': 'aquasec/trivy:0.18.3',
'commands': commands,
}
def scan_docker_image_high_critical_vulnerabilities_step(edition):
tags=['latest', 'main', 'latest-ubuntu', 'main-ubuntu']
commands=[]
for t in tags:
commands.append('trivy --exit-code 1 --severity HIGH,CRITICAL grafana/{}:{}'.format(edition,t))
return {
'name': 'scan-docker-images-high-critical-vulnerabilities',
'image': 'aquasec/trivy:0.18.3',
'commands': commands,
}
def slack_job_failed_step(channel):
return {
'name': 'slack-notify-failure',
'image': 'plugins/slack',
'settings': {
'webhook': from_secret('slack_webhook_backend'),
'channel': channel,
'template': 'Nightly docker image scan job for {{repo.name}} failed: {{build.link}}',
},
'when': {
'status': 'failure'
}
}
+181
View File
@@ -0,0 +1,181 @@
load(
'scripts/drone/steps/lib.star',
'lint_backend_step',
'codespell_step',
'shellcheck_step',
'test_backend_step',
'test_frontend_step',
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'gen_version_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
'build_storybook_step',
'build_frontend_docs_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
'memcached_integration_tests_step',
'get_windows_steps',
'benchmark_ldap_step',
'enterprise_downstream_step',
'frontend_metrics_step',
'publish_storybook_step',
'release_canary_npm_packages_step',
'upload_packages_step',
'push_to_deployment_tools_step',
'publish_packages_step',
'upload_cdn_step',
'validate_scuemata_step',
'test_a11y_frontend_step'
)
load(
'scripts/drone/services/services.star',
'integration_test_services',
'ldap_service',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
'notify_pipeline',
)
ver_mode = 'main'
def get_steps(edition, is_downstream=False):
publish = edition != 'enterprise' or is_downstream
include_enterprise2 = edition == 'enterprise'
steps = [
enterprise_downstream_step(edition=edition),
codespell_step(),
shellcheck_step(),
lint_backend_step(edition=edition),
test_backend_step(edition=edition),
test_frontend_step(),
build_backend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_frontend_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
build_plugins_step(edition=edition, sign=True),
validate_scuemata_step(),
]
# Have to insert Enterprise2 steps before they're depended on (in the gen-version step)
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
lint_backend_step(edition=edition2),
test_backend_step(edition=edition2),
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream),
])
# Insert remaining steps
steps.extend([
gen_version_step(ver_mode=ver_mode, is_downstream=is_downstream, include_enterprise2=include_enterprise2),
package_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
e2e_tests_server_step(edition=edition),
e2e_tests_step(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
publish_storybook_step(edition=edition, ver_mode=ver_mode),
test_a11y_frontend_step(edition=edition),
frontend_metrics_step(edition=edition),
build_frontend_docs_step(edition=edition),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=publish),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=publish),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
])
if include_enterprise2:
steps.extend([redis_integration_tests_step(), memcached_integration_tests_step()])
steps.extend([
release_canary_npm_packages_step(edition),
upload_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
push_to_deployment_tools_step(edition=edition, is_downstream=is_downstream),
upload_cdn_step(edition=edition)
])
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
package_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64'], is_downstream=is_downstream),
e2e_tests_server_step(edition=edition2, port=3002),
e2e_tests_step(edition=edition2, port=3002),
upload_packages_step(edition=edition2, ver_mode=ver_mode, is_downstream=is_downstream),
upload_cdn_step(edition=edition2)
])
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream)
if edition == 'enterprise' and not is_downstream:
publish_steps = []
else:
publish_steps = [
publish_packages_step(edition=edition, ver_mode=ver_mode, is_downstream=is_downstream),
]
return steps, windows_steps, publish_steps
def main_pipelines(edition):
services = integration_test_services(edition)
trigger = {
'event': ['push',],
'branch': 'main',
}
steps, windows_steps, publish_steps = get_steps(edition=edition)
if edition == 'enterprise':
steps.append(benchmark_ldap_step())
services.append(ldap_service())
pipelines = [
pipeline(
name='build-main', edition=edition, trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
),
pipeline(
name='windows-main', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
depends_on=['build-main'], ver_mode=ver_mode,
),
]
if edition != 'enterprise':
pipelines.append(pipeline(
name='publish-main', edition=edition, trigger=trigger, steps=publish_steps,
depends_on=['build-main', 'windows-main',], install_deps=False, ver_mode=ver_mode,
))
pipelines.append(notify_pipeline(
name='notify-main', slack_channel='grafana-ci-notifications', trigger=trigger,
depends_on=['build-main', 'windows-main', 'publish-main'],
))
else:
# Add downstream enterprise pipelines triggerable from OSS builds
trigger = {
'event': ['custom',],
}
steps, windows_steps, publish_steps = get_steps(edition=edition, is_downstream=True)
pipelines.append(pipeline(
name='build-main-downstream', edition=edition, trigger=trigger, services=services, steps=steps,
is_downstream=True, ver_mode=ver_mode,
))
pipelines.append(pipeline(
name='windows-main-downstream', edition=edition, trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['build-main-downstream'], is_downstream=True, ver_mode=ver_mode,
))
pipelines.append(pipeline(
name='publish-main-downstream', edition=edition, trigger=trigger, steps=publish_steps,
depends_on=['build-main-downstream', 'windows-main-downstream'], is_downstream=True, install_deps=False,
ver_mode=ver_mode,
))
pipelines.append(notify_pipeline(
name='notify-main-downstream', slack_channel='grafana-enterprise-ci-notifications', trigger=trigger,
depends_on=['build-main-downstream', 'windows-main-downstream', 'publish-main-downstream'],
))
return pipelines
+100
View File
@@ -0,0 +1,100 @@
load(
'scripts/drone/steps/lib.star',
'lint_backend_step',
'codespell_step',
'shellcheck_step',
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'test_backend_step',
'test_frontend_step',
'gen_version_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
'build_storybook_step',
'build_frontend_docs_step',
'build_docs_website_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
'memcached_integration_tests_step',
'benchmark_ldap_step',
'validate_scuemata_step',
)
load(
'scripts/drone/services/services.star',
'integration_test_services',
'ldap_service',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
)
ver_mode = 'pr'
def pr_pipelines(edition):
services = integration_test_services(edition)
variants = ['linux-x64', 'linux-x64-musl', 'osx64', 'win64',]
include_enterprise2 = edition == 'enterprise'
steps = [
codespell_step(),
shellcheck_step(),
lint_backend_step(edition=edition),
test_backend_step(edition=edition),
test_frontend_step(),
build_backend_step(edition=edition, ver_mode=ver_mode, variants=variants),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition),
validate_scuemata_step(),
]
# Have to insert Enterprise2 steps before they're depended on (in the gen-version step)
if include_enterprise2:
edition2 = 'enterprise2'
steps.append(benchmark_ldap_step())
services.append(ldap_service())
steps.extend([
lint_backend_step(edition=edition2),
test_backend_step(edition=edition2),
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
])
# Insert remaining steps
steps.extend([
gen_version_step(ver_mode=ver_mode, include_enterprise2=include_enterprise2),
package_step(edition=edition, ver_mode=ver_mode, variants=variants),
e2e_tests_server_step(edition=edition),
e2e_tests_step(edition=edition),
build_storybook_step(edition=edition, ver_mode=ver_mode),
build_frontend_docs_step(edition=edition),
build_docs_website_step(),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, archs=['amd64',]),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
])
if include_enterprise2:
steps.extend([
redis_integration_tests_step(),
memcached_integration_tests_step(),
package_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
e2e_tests_server_step(edition=edition2, port=3002),
e2e_tests_step(edition=edition2, port=3002),
])
trigger = {
'event': ['pull_request',],
}
return [
pipeline(
name='test-pr', edition=edition, trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
),
]
+233
View File
@@ -0,0 +1,233 @@
load(
'scripts/drone/steps/lib.star',
'test_release_ver',
'build_image',
'publish_image',
'lint_backend_step',
'codespell_step',
'shellcheck_step',
'test_backend_step',
'test_frontend_step',
'build_backend_step',
'build_frontend_step',
'build_plugins_step',
'gen_version_step',
'package_step',
'e2e_tests_server_step',
'e2e_tests_step',
'build_storybook_step',
'copy_packages_for_docker_step',
'build_docker_images_step',
'postgres_integration_tests_step',
'mysql_integration_tests_step',
'redis_integration_tests_step',
'memcached_integration_tests_step',
'get_windows_steps',
'benchmark_ldap_step',
'frontend_metrics_step',
'publish_storybook_step',
'upload_packages_step',
'publish_packages_step',
'upload_cdn_step',
'validate_scuemata_step'
)
load(
'scripts/drone/services/services.star',
'integration_test_services',
'ldap_service',
)
load(
'scripts/drone/utils/utils.star',
'pipeline',
'notify_pipeline',
)
def release_npm_packages_step(edition, ver_mode):
if edition == 'enterprise':
return None
if ver_mode == 'release':
commands = ['./scripts/build/release-packages.sh ${DRONE_TAG}']
else:
commands = []
return {
'name': 'release-npm-packages',
'image': build_image,
'depends_on': [
# Has to run after publish-storybook since this step cleans the files publish-storybook depends on
'publish-storybook',
],
'environment': {
'NPM_TOKEN': {
'from_secret': 'npm_token',
},
'GITHUB_PACKAGE_TOKEN': {
'from_secret': 'github_package_token',
},
},
'commands': commands,
}
def get_steps(edition, ver_mode):
should_publish = ver_mode in ('release', 'test-release',)
should_upload = should_publish or ver_mode in ('release-branch',)
include_enterprise2 = edition == 'enterprise'
tries = None
if should_publish:
tries = 5
steps = [
codespell_step(),
shellcheck_step(),
lint_backend_step(edition=edition),
test_backend_step(edition=edition, tries=tries),
test_frontend_step(),
build_backend_step(edition=edition, ver_mode=ver_mode),
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, sign=True),
validate_scuemata_step(),
]
# Have to insert Enterprise2 steps before they're depended on (in the gen-version step)
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
lint_backend_step(edition=edition2),
test_backend_step(edition=edition2, tries=tries),
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
])
# Insert remaining steps
steps.extend([
gen_version_step(ver_mode=ver_mode, include_enterprise2=include_enterprise2),
package_step(edition=edition, ver_mode=ver_mode),
e2e_tests_server_step(edition=edition),
e2e_tests_step(edition=edition, tries=3),
build_storybook_step(edition=edition, ver_mode=ver_mode),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=should_publish),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=should_publish),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
])
if include_enterprise2:
steps.extend([redis_integration_tests_step(), memcached_integration_tests_step()])
if should_upload:
steps.append(upload_cdn_step(edition=edition))
steps.append(upload_packages_step(edition=edition, ver_mode=ver_mode))
if should_publish:
steps.extend([
publish_storybook_step(edition=edition, ver_mode=ver_mode),
release_npm_packages_step(edition=edition, ver_mode=ver_mode),
])
windows_steps = get_windows_steps(edition=edition, ver_mode=ver_mode)
if include_enterprise2:
edition2 = 'enterprise2'
steps.extend([
package_step(edition=edition2, ver_mode=ver_mode, variants=['linux-x64']),
e2e_tests_server_step(edition=edition2, port=3002),
e2e_tests_step(edition=edition2, port=3002, tries=3),
upload_cdn_step(edition=edition2),
])
if should_upload:
steps.append(upload_packages_step(edition=edition2, ver_mode=ver_mode))
return steps, windows_steps
def get_oss_pipelines(trigger, ver_mode):
services = integration_test_services(edition='oss')
steps, windows_steps = get_steps(edition='oss', ver_mode=ver_mode)
return [
pipeline(
name='oss-build-{}'.format(ver_mode), edition='oss', trigger=trigger, services=services, steps=steps,
ver_mode=ver_mode,
),
pipeline(
name='oss-windows-{}'.format(ver_mode), edition='oss', trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['oss-build-{}'.format(ver_mode)], ver_mode=ver_mode,
),
]
def get_enterprise_pipelines(trigger, ver_mode):
services = integration_test_services(edition='enterprise')
steps, windows_steps = get_steps(edition='enterprise', ver_mode=ver_mode)
return [
pipeline(
name='enterprise-build-{}'.format(ver_mode), edition='enterprise', trigger=trigger, services=services,
steps=steps, ver_mode=ver_mode,
),
pipeline(
name='enterprise-windows-{}'.format(ver_mode), edition='enterprise', trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['enterprise-build-{}'.format(ver_mode)], ver_mode=ver_mode,
),
]
def release_pipelines(ver_mode='release', trigger=None):
# 'enterprise' edition services contain both OSS and enterprise services
services = integration_test_services(edition='enterprise')
if not trigger:
trigger = {
'ref': ['refs/tags/v*',],
}
should_publish = ver_mode in ('release', 'test-release',)
# The release pipelines include also enterprise ones, so both editions are built for a release.
# We could also solve this by triggering a downstream build for the enterprise repo, but by including enterprise
# in OSS release builds, we simplify the UX for the release engineer.
oss_pipelines = get_oss_pipelines(ver_mode=ver_mode, trigger=trigger)
enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger)
pipelines = oss_pipelines + enterprise_pipelines
if should_publish:
publish_pipeline = pipeline(
name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', steps=[
publish_packages_step(edition='oss', ver_mode=ver_mode),
publish_packages_step(edition='enterprise', ver_mode=ver_mode),
], depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], install_deps=False,
ver_mode=ver_mode,
)
pipelines.append(publish_pipeline)
pipelines.append(notify_pipeline(
name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=trigger,
depends_on=[p['name'] for p in pipelines],
))
return pipelines
def test_release_pipelines():
ver_mode = 'test-release'
services = integration_test_services(edition='enterprise')
trigger = {
'event': ['custom',],
}
oss_pipelines = get_oss_pipelines(ver_mode=ver_mode, trigger=trigger)
enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger)
publish_cmd = './bin/grabpl publish-packages --edition {{}} --dry-run {}'.format(test_release_ver)
publish_pipeline = pipeline(
name='publish-{}'.format(ver_mode), trigger=trigger, edition='oss', steps=[
publish_packages_step(edition='oss', ver_mode=ver_mode),
publish_packages_step(edition='enterprise', ver_mode=ver_mode),
], depends_on=[p['name'] for p in oss_pipelines + enterprise_pipelines], install_deps=False,
ver_mode=ver_mode,
)
pipelines = oss_pipelines + enterprise_pipelines + [publish_pipeline,]
pipelines.append(notify_pipeline(
name='notify-{}'.format(ver_mode), slack_channel='grafana-ci-notifications', trigger=trigger,
depends_on=[p['name'] for p in pipelines],
))
return pipelines