CI: Migrate to Drone for master builds (#27087)

* CI: Migrate to Drone for master builds
This commit is contained in:
Arve Knudsen
2020-09-24 15:20:48 +02:00
committed by GitHub
parent b98c85d81c
commit 755f3d0de7
3 changed files with 418 additions and 225 deletions
+278 -94
View File
@@ -1,9 +1,12 @@
build_image = 'grafana/build-container:1.2.27'
publish_image = 'grafana/grafana-ci-deploy:1.2.6'
grafana_docker_image = 'grafana/drone-grafana-docker:0.2.0'
grafana_docker_image = 'grafana/drone-grafana-docker:0.3.1'
alpine_image = 'alpine:3.12'
windows_image = 'mcr.microsoft.com/windows:1809'
grabpl_version = '0.5.9'
grabpl_version = '0.5.10'
git_image = 'alpine/git:v2.26.2'
dockerize_version = '0.6.1'
wix_image = 'grafana/ci-wix:0.1.1'
def pr_pipelines(edition):
services = [
@@ -47,14 +50,51 @@ def pr_pipelines(edition):
postgres_integration_tests_step(),
mysql_integration_tests_step(),
]
windows_steps = get_windows_steps(edition=edition, version_mode='pr')
trigger = {
'event': ['pull_request',],
}
return [
pipeline(
name='test-pr', edition=edition, trigger={
'event': ['pull_request',],
}, services=services, steps=steps
name='test-pr', edition=edition, trigger=trigger, services=services, steps=steps
),
pipeline(
name='windows-pr', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
),
]
def master_steps(edition, is_downstream=False):
publish = edition != 'enterprise' or is_downstream
steps = [
enterprise_downstream_step(edition),
lint_backend_step(edition),
codespell_step(),
shellcheck_step(),
test_backend_step(),
test_frontend_step(),
frontend_metrics_step(edition=edition),
build_backend_step(edition=edition, is_downstream=is_downstream),
build_frontend_step(edition=edition, is_downstream=is_downstream),
build_plugins_step(edition=edition, sign=True),
package_step(edition=edition, sign=True, is_downstream=is_downstream),
e2e_tests_server_step(),
e2e_tests_step(),
build_storybook_step(edition=edition),
publish_storybook_step(edition=edition),
build_docs_website_step(),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition, publish=publish),
build_docker_images_step(edition=edition, ubuntu=True, publish=publish),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
release_next_npm_packages_step(edition),
publish_packages_step(edition, is_downstream),
deploy_to_kubernetes_step(edition, is_downstream),
]
windows_steps = get_windows_steps(edition=edition, version_mode='master', is_downstream=is_downstream)
return steps, windows_steps
def master_pipelines(edition):
services = [
{
@@ -77,49 +117,38 @@ def master_pipelines(edition):
},
},
]
steps = [
lint_backend_step(edition),
codespell_step(),
shellcheck_step(),
test_backend_step(),
test_frontend_step(),
frontend_metrics_step(),
build_backend_step(edition=edition),
build_frontend_step(edition=edition),
build_plugins_step(edition=edition),
package_step(edition=edition),
e2e_tests_server_step(),
e2e_tests_step(),
build_storybook_step(edition=edition),
publish_storybook_step(edition=edition),
build_docs_website_step(),
copy_packages_for_docker_step(),
build_docker_images_step(edition=edition),
build_docker_images_step(edition=edition, ubuntu=True),
postgres_integration_tests_step(),
mysql_integration_tests_step(),
release_next_npm_packages_step(edition),
publish_packages_step(edition),
deploy_to_kubernetes_step(edition),
]
windows_steps = [
windows_installer_step(edition),
]
trigger = {
'event': ['push',],
'branch': 'master',
}
return [
steps, windows_steps = master_steps(edition=edition)
pipelines = [
pipeline(
name='test-master', edition=edition, trigger=trigger, services=services, steps=steps
name='build-master', edition=edition, trigger=trigger, services=services, steps=steps
),
pipeline(
name='windows-installer-master', edition=edition, trigger=trigger,
steps=windows_steps, platform='windows', depends_on=['test-master'],
name='windows-master', edition=edition, trigger=trigger, steps=windows_steps, platform='windows',
depends_on=['build-master'],
),
]
if edition == 'enterprise':
# Add downstream enterprise pipelines triggerable from OSS builds
trigger = {
'event': ['custom',],
}
steps, windows_steps = master_steps(edition=edition, is_downstream=True)
pipelines.append(pipeline(
name='build-master-downstream', edition=edition, trigger=trigger, services=services, steps=steps,
is_downstream=True,
))
pipelines.append(pipeline(
name='windows-master-downstream', edition=edition, trigger=trigger, steps=windows_steps,
platform='windows', depends_on=['build-master-downstream'], is_downstream=True,
))
def pipeline(name, edition, trigger, steps, services=[], platform='linux', depends_on=[]):
return pipelines
def pipeline(name, edition, trigger, steps, services=[], platform='linux', depends_on=[], is_downstream=False):
if platform != 'windows':
platform_conf = {
'os': 'linux',
@@ -139,7 +168,7 @@ def pipeline(name, edition, trigger, steps, services=[], platform='linux', depen
'name': name,
'trigger': trigger,
'services': services,
'steps': init_steps(edition, platform) + steps,
'steps': init_steps(edition, platform, is_downstream=is_downstream) + steps,
'depends_on': depends_on,
}
@@ -151,14 +180,14 @@ def pipeline(name, edition, trigger, steps, services=[], platform='linux', depen
return pipeline
def init_steps(edition, platform):
def init_steps(edition, platform, is_downstream=False):
if platform == 'windows':
return [
{
'name': 'identify-runner',
'image': windows_image,
'commands': [
'echo $Env:DRONE_RUNNER_NAME',
'echo $env:DRONE_RUNNER_NAME',
],
},
]
@@ -178,11 +207,15 @@ def init_steps(edition, platform):
'yarn install --frozen-lockfile --no-progress',
]
if edition == 'enterprise':
if is_downstream:
source_commit = ' $${SOURCE_COMMIT}'
else:
source_commit = ''
return [
identify_runner_step,
{
'name': 'clone',
'image': 'alpine/git:v2.26.2',
'image': git_image,
'environment': {
'GITHUB_TOKEN': {
'from_secret': 'github_token',
@@ -198,18 +231,19 @@ def init_steps(edition, platform):
'name': 'initialize',
'image': build_image,
'environment': {
'GRABPL_VERSION': grabpl_version,
'DOCKERIZE_VERSION': '0.6.1',
'DOCKERIZE_VERSION': dockerize_version,
},
'depends_on': [
'clone',
],
'commands': [
'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl',
'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/grabpl'.format(
grabpl_version
),
'chmod +x grabpl',
'mv grabpl /tmp',
'mv grafana-enterprise /tmp/',
'/tmp/grabpl init-enterprise /tmp/grafana-enterprise',
'/tmp/grabpl init-enterprise /tmp/grafana-enterprise{}'.format(source_commit),
'mkdir bin',
'mv /tmp/grabpl bin/'
] + common_cmds,
@@ -222,11 +256,12 @@ def init_steps(edition, platform):
'name': 'initialize',
'image': build_image,
'environment': {
'GRABPL_VERSION': grabpl_version,
'DOCKERIZE_VERSION': '0.6.1',
'DOCKERIZE_VERSION': dockerize_version,
},
'commands': [
'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v$${GRABPL_VERSION}/grabpl',
'curl -fLO https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/grabpl'.format(
grabpl_version
),
'chmod +x grabpl',
'mkdir -p bin',
'mv grabpl bin',
@@ -234,6 +269,28 @@ def init_steps(edition, platform):
},
]
def enterprise_downstream_step(edition):
if edition == 'enterprise':
return None
return {
'name': 'trigger-enterprise-downstream',
'image': 'grafana/drone-downstream',
'settings': {
'server': 'https://drone.grafana.net',
'token': {
'from_secret': 'drone_token',
},
'repositories': [
'grafana/grafana-enterprise',
],
'params': [
'SOURCE_BUILD_NUMBER=${DRONE_BUILD_NUMBER}',
'SOURCE_COMMIT=${DRONE_COMMIT}',
],
},
}
def lint_backend_step(edition):
return {
'name': 'lint-backend',
@@ -285,11 +342,15 @@ def publish_storybook_step(edition):
'commands': [
'printenv GCP_KEY | base64 -d > /tmp/gcpkey.json',
'gcloud auth activate-service-account --key-file=/tmp/gcpkey.json',
'echo gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary',
'gsutil -m rsync -d -r ./packages/grafana-ui/dist/storybook gs://grafana-storybook/canary',
],
}
def build_backend_step(edition, variants=None):
def build_backend_step(edition, variants=None, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
if variants:
variants_str = ' --variants {}'.format(','.join(variants))
else:
@@ -304,13 +365,17 @@ def build_backend_step(edition, variants=None):
],
'commands': [
# TODO: Convert number of jobs to percentage
'./bin/grabpl build-backend --jobs 8 --edition {} --build-id $DRONE_BUILD_NUMBER{} --no-pull-enterprise'.format(
edition, variants_str
'./bin/grabpl build-backend --jobs 8 --edition {} --build-id {}{} --no-pull-enterprise'.format(
edition, build_no, variants_str,
),
],
}
def build_frontend_step(edition):
def build_frontend_step(edition, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
return {
'name': 'build-frontend',
'image': build_image,
@@ -321,11 +386,21 @@ def build_frontend_step(edition):
'commands': [
# TODO: Use percentage for num jobs
'./bin/grabpl build-frontend --jobs 8 --no-install-deps --edition {} '.format(edition) +
'--build-id $DRONE_BUILD_NUMBER --no-pull-enterprise',
'--build-id {} --no-pull-enterprise'.format(build_no),
],
}
def build_plugins_step(edition):
def build_plugins_step(edition, sign=False):
if sign:
env = {
'GRAFANA_API_KEY': {
'from_secret': 'grafana_api_key',
},
}
sign_args = ' --sign --signing-admin'
else:
env = None
sign_args = ''
return {
'name': 'build-plugins',
'image': build_image,
@@ -333,9 +408,10 @@ def build_plugins_step(edition):
'initialize',
'lint-backend',
],
'environment': env,
'commands': [
# TODO: Use percentage for num jobs
'./bin/grabpl build-plugins --jobs 8 --edition {} --no-install-deps'.format(edition),
'./bin/grabpl build-plugins --jobs 8 --edition {} --no-install-deps{}'.format(edition, sign_args),
],
}
@@ -370,7 +446,10 @@ def test_frontend_step():
],
}
def frontend_metrics_step():
def frontend_metrics_step(edition):
if edition == 'enterprise':
return None
return {
'name': 'frontend-metrics',
'image': build_image,
@@ -425,11 +504,37 @@ def shellcheck_step():
],
}
def package_step(edition, variants=None):
def package_step(edition, variants=None, sign=False, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
if variants:
variants_str = ' --variants {}'.format(','.join(variants))
else:
variants_str = ''
if sign:
sign_args = ' --sign'
env = {
'GRAFANA_API_KEY': {
'from_secret': 'grafana_api_key',
},
'GPG_PRIV_KEY': {
'from_secret': 'gpg_priv_key',
},
'GPG_PUB_KEY': {
'from_secret': 'gpg_pub_key',
},
'GPG_KEY_PASSWORD': {
'from_secret': 'gpg_key_password',
},
}
test_args = ''
else:
sign_args = ''
env = None
test_args = '. scripts/build/gpg-test-vars.sh && '
return {
'name': 'package',
'image': build_image,
@@ -442,10 +547,11 @@ def package_step(edition, variants=None):
'codespell',
'shellcheck',
],
'environment': env,
'commands': [
# TODO: Use percentage for jobs
'. scripts/build/gpg-test-vars.sh && ./bin/grabpl package --jobs 8 --edition {} '.format(edition) +
'--build-id $DRONE_BUILD_NUMBER --no-pull-enterprise' + variants_str,
'{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) +
'--build-id {} --no-pull-enterprise{}{}'.format(build_no, variants_str, sign_args),
],
}
@@ -507,15 +613,22 @@ def copy_packages_for_docker_step():
],
}
def build_docker_images_step(edition, archs=None, ubuntu=False):
def build_docker_images_step(edition, archs=None, ubuntu=False, publish=False):
sfx = ''
if ubuntu:
sfx = '-ubuntu'
settings = {
'dry_run': True,
'dry_run': not publish,
'edition': edition,
'ubuntu': ubuntu,
}
if publish:
settings['username'] = {
'from_secret': 'docker_user',
}
settings['password'] = {
'from_secret': 'docker_password',
}
if archs:
settings['archs'] = ','.join(archs)
return {
@@ -593,13 +706,12 @@ def release_next_npm_packages_step(edition):
'commands': [
'npx lerna bootstrap',
'echo "//registry.npmjs.org/:_authToken=$${NPM_TOKEN}" >> ~/.npmrc',
# TODO: Enable
'echo ./scripts/circle-release-next-packages.sh',
'./scripts/circle-release-next-packages.sh',
],
}
def deploy_to_kubernetes_step(edition):
if edition != 'enterprise':
def deploy_to_kubernetes_step(edition, is_downstream):
if edition != 'enterprise' or not is_downstream:
return None
return {
@@ -608,13 +720,20 @@ def deploy_to_kubernetes_step(edition):
'depends_on': [
'build-docker-images',
],
'environment': {
'CIRCLE_TOKEN': {
'from_secret': 'deployment_tools_circle_token',
},
},
'commands': [
# TODO: Enable
'echo ./bin/grabpl deploy-to-k8s',
'./bin/grabpl deploy-to-k8s',
],
}
def publish_packages_step(edition):
def publish_packages_step(edition, is_downstream):
if edition == 'enterprise' and not is_downstream:
return None
return {
'name': 'publish-packages',
'image': publish_image,
@@ -624,39 +743,104 @@ def publish_packages_step(edition):
'mysql-integration-tests',
'postgres-integration-tests',
],
'commands': [
# TODO: Enable
'echo ./bin/grabpl publish-packages --edition {}'.format(edition),
],
}
def windows_installer_step(edition):
sfx = ''
if edition == 'enterprise':
sfx = '-enterprise'
return {
'name': 'build-windows-installer',
'image': 'grafana/ci-wix:0.1.1',
'environment': {
'GCP_KEY': {
'GCP_GRAFANA_UPLOAD_KEY': {
'from_secret': 'gcp_key',
},
'GRAFANA_COM_API_KEY': {
'from_secret': 'grafana_api_key',
},
},
'commands': [
'./bin/grabpl publish-packages --edition {}'.format(edition),
],
}
def get_windows_steps(edition, version_mode, is_downstream=False):
if not is_downstream:
source_commit = ''
else:
source_commit = ' $$env:SOURCE_COMMIT'
sfx = ''
if edition == 'enterprise':
sfx = '-enterprise'
if not is_downstream:
build_no = 'DRONE_BUILD_NUMBER'
else:
build_no = 'SOURCE_BUILD_NUMBER'
steps = [
{
'name': 'initialize',
'image': wix_image,
'commands': [
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version),
],
},
]
if version_mode == 'master':
installer_commands = [
'$$gcpKey = $$env:GCP_KEY',
'[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json',
# gcloud fails to read the file unless converted with dos2unix
'dos2unix gcpkey.json',
'gcloud auth activate-service-account --key-file=gcpkey.json',
'rm gcpkey.json',
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version),
# TODO: Infer correct Grafana version
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/{}/master/grafana{}-7.2.0-9fffe273pre.windows-amd64.zip -OutFile grafana.zip'.format(edition, sfx),
'cp C:\\App\\nssm-2.24.zip .',
'./grabpl.exe windows-installer --edition {} grafana.zip'.format(edition),
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
'echo "gsutil cp $$fname gs://grafana-downloads/{}/master/"'.format(edition),
'echo "gsutil cp $$fname.sha256 gs://grafana-downloads/{}/master/"'.format(edition),
],
}
'.\\grabpl.exe windows-installer --edition {} --build-id $$env:{}'.format(edition, build_no),
]
if edition != 'enterprise' or is_downstream:
installer_commands.extend([
'$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
'gsutil cp $$fname gs://grafana-downloads/{}/{}/'.format(edition, version_mode),
'gsutil cp $$fname.sha256 gs://grafana-downloads/{}/{}/'.format(edition, version_mode),
])
steps.append({
'name': 'build-windows-installer',
'image': wix_image,
'environment': {
'GCP_KEY': {
'from_secret': 'gcp_key',
},
},
'commands': installer_commands,
'depends_on': [
'initialize',
],
})
if edition == 'enterprise':
# For enterprise, we have to clone both OSS and enterprise and merge the latter into the former
clone_commands = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
]
if not is_downstream:
clone_commands.extend([
'cd grafana-enterprise',
'git checkout $$env:DRONE_COMMIT',
])
steps.insert(0, {
'name': 'clone',
'image': wix_image,
'environment': {
'GITHUB_TOKEN': {
'from_secret': 'github_token',
},
},
'commands': clone_commands,
})
steps[1]['depends_on'] = [
'clone',
]
steps[1]['commands'].extend([
# Need to move grafana-enterprise out of the way, so directory is empty and can be cloned into
'cp -r grafana-enterprise C:\\App\\grafana-enterprise',
'rm -r -force grafana-enterprise',
'cp grabpl.exe C:\\App\\grabpl.exe',
'rm -force grabpl.exe',
'C:\\App\\grabpl.exe init-enterprise C:\\App\\grafana-enterprise{}'.format(source_commit),
'cp C:\\App\\grabpl.exe grabpl.exe',
])
return steps