public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: Split enterprise2 pipelines (#57997)

Browse Source 
* Add publishimages_enterprise2.go file

* Add push.go

* Modify build files

* Modify starlark files

* Remove promotion pipeline

* Remove unused args

* Handle tag for custom events
This commit is contained in:
Dimitris Sotirakis
2022-11-21 20:06:00 +02:00
committed by GitHub
parent 1f4834a144
commit 44402ff726
12 changed files with 1084 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/events/release.star
+84 -24
View File
@@ -73,6 +73,16 @@ load(
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
ver_mode='release'
release_trigger = {
'event': {
'exclude': [
'promote'
]
},
'ref': ['refs/tags/v*',],
}
def store_npm_packages_step():
return {
'name': 'store-npm-packages',
@@ -122,7 +132,7 @@ def release_npm_packages_step():
],
}
def get_oss_pipelines(trigger, ver_mode):
def oss_pipelines(ver_mode=ver_mode, trigger=release_trigger):
environment = {'EDITION': 'oss'}
edition = 'oss'
services = integration_test_services(edition=edition)
@@ -223,7 +233,7 @@ def get_oss_pipelines(trigger, ver_mode):
pipelines.extend([windows_pipeline])
return pipelines
def get_enterprise_pipelines(trigger, ver_mode):
def enterprise_pipelines(ver_mode=ver_mode, trigger=release_trigger):
environment = {'EDITION': 'enterprise'}
edition = 'enterprise'
services = integration_test_services(edition=edition)
@@ -350,6 +360,78 @@ def get_enterprise_pipelines(trigger, ver_mode):
return pipelines
def enterprise2_pipelines(prefix='', ver_mode=ver_mode, trigger=release_trigger):
environment = {
'EDITION': 'enterprise2',
}
edition = 'enterprise'
services = integration_test_services(edition=edition)
volumes = integration_test_services_volumes()
package_steps = []
publish_steps = []
should_publish = ver_mode == 'release'
should_upload = should_publish or ver_mode in ('release-branch',)
include_enterprise = edition == 'enterprise'
edition2 = 'enterprise2'
init_steps = [
download_grabpl_step(),
identify_runner_step(),
clone_enterprise_step(ver_mode),
init_enterprise_step(ver_mode),
compile_build_cmd(edition),
]
build_steps = [
build_frontend_step(edition=edition, ver_mode=ver_mode),
build_frontend_package_step(edition=edition, ver_mode=ver_mode),
build_plugins_step(edition=edition, ver_mode=ver_mode),
]
if include_enterprise:
build_steps.extend([
build_backend_step(edition=edition2, ver_mode=ver_mode, variants=['linux-amd64']),
])
fetch_images = fetch_images_step(edition2)
fetch_images.update({'depends_on': ['build-docker-images', 'build-docker-images-ubuntu']})
upload_cdn = upload_cdn_step(edition=edition2, ver_mode=ver_mode)
upload_cdn['environment'].update({'ENTERPRISE2_CDN_PATH': from_secret('enterprise2-cdn-path')})
build_steps.extend([
package_step(edition=edition2, ver_mode=ver_mode, include_enterprise2=include_enterprise, variants=['linux-amd64']),
upload_cdn,
copy_packages_for_docker_step(edition=edition2),
build_docker_images_step(edition=edition2, ver_mode=ver_mode, publish=True),
build_docker_images_step(edition=edition2, ver_mode=ver_mode, ubuntu=True, publish=True),
fetch_images,
publish_images_step(edition2, 'release', mode=edition2, docker_repo='${{DOCKER_ENTERPRISE2_REPO}}'),
])
if should_upload:
step = upload_packages_step(edition=edition2, ver_mode=ver_mode)
if step:
publish_steps.append(step)
deps_on_clone_enterprise_step = {
'depends_on': [
'init-enterprise',
]
}
for step in [wire_install_step(), yarn_install_step(), verify_gen_cue_step(edition)]:
step.update(deps_on_clone_enterprise_step)
init_steps.extend([step])
pipelines = [
pipeline(
name='{}{}-enterprise2-build{}-publish'.format(prefix, ver_mode, get_e2e_suffix()), edition=edition, trigger=trigger, services=[],
steps=init_steps + build_steps + package_steps + publish_steps,
volumes=volumes, environment=environment,
),
]
return pipelines
def publish_artifacts_step(mode):
security = ''
if mode == 'security':
@@ -435,28 +517,6 @@ def artifacts_page_pipeline():
return [pipeline(name='publish-artifacts-page', trigger=trigger, steps = [download_grabpl_step(), artifacts_page_step()], edition="all", environment = {'EDITION': 'all'}
)]
def release_pipelines(ver_mode='release', trigger=None):
# 'enterprise' edition services contain both OSS and enterprise services
if not trigger:
trigger = {
'event': {
'exclude': [
'promote'
]
},
'ref': ['refs/tags/v*',],
}
# The release pipelines include also enterprise ones, so both editions are built for a release.
# We could also solve this by triggering a downstream build for the enterprise repo, but by including enterprise
# in OSS release builds, we simplify the UX for the release engineer.
oss_pipelines = get_oss_pipelines(ver_mode=ver_mode, trigger=trigger)
enterprise_pipelines = get_enterprise_pipelines(ver_mode=ver_mode, trigger=trigger)
pipelines = oss_pipelines + enterprise_pipelines
return pipelines
def get_e2e_suffix():
if not disable_tests:
return '-e2e'
scripts/drone/pipelines/build.star
+2 -2
View File
@@ -93,8 +93,8 @@ def build_e2e(trigger, ver_mode, edition):
build_steps.extend([
build_docker_images_step(edition=edition, ver_mode=ver_mode, publish=False),
build_docker_images_step(edition=edition, ver_mode=ver_mode, ubuntu=True, publish=False),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana/grafana', trigger=trigger_oss),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana/grafana-oss', trigger=trigger_oss),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana', trigger=trigger_oss),
publish_images_step(edition=edition, ver_mode=ver_mode, mode='', docker_repo='grafana-oss', trigger=trigger_oss),
release_canary_npm_packages_step(edition, trigger=trigger_oss),
upload_packages_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss),
upload_cdn_step(edition=edition, ver_mode=ver_mode, trigger=trigger_oss)
scripts/drone/pipelines/publish_images.star
+3 -3
View File
@@ -34,9 +34,9 @@ def publish_image_pipelines_public():
'target': [mode],
}
return [pipeline(
name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana/grafana'), edition="", environment = {'EDITION': 'oss'}
name='publish-docker-oss-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='oss', mode=mode, docker_repo='grafana'), edition="", environment = {'EDITION': 'oss'}
), pipeline(
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana/grafana-enterprise'), edition="", environment = {'EDITION': 'enterprise'}
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition="", environment = {'EDITION': 'enterprise'}
),]
def publish_image_pipelines_security():
@@ -46,5 +46,5 @@ def publish_image_pipelines_security():
'target': [mode],
}
return [pipeline(
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana/grafana-enterprise'), edition="", environment = {'EDITION': 'enterprise'}
name='publish-docker-enterprise-{}'.format(mode), trigger=trigger, steps=publish_image_steps(edition='enterprise', mode=mode, docker_repo='grafana-enterprise'), edition="", environment = {'EDITION': 'enterprise'}
),]
scripts/drone/steps/lib.star
+29 -19
View File
@@ -622,15 +622,10 @@ def codespell_step():
def package_step(edition, ver_mode, include_enterprise2=False, variants=None):
deps = [
'build-plugins',
'build-backend',
'build-backend' + enterprise2_suffix(edition),
'build-frontend',
'build-frontend-packages',
]
if include_enterprise2:
sfx = '-enterprise2'
deps.extend([
'build-backend' + sfx,
])
variants_str = ''
if variants:
@@ -773,12 +768,12 @@ def build_docs_website_step():
}
def copy_packages_for_docker_step():
def copy_packages_for_docker_step(edition=None):
return {
'name': 'copy-packages-for-docker',
'image': build_image,
'depends_on': [
'package',
'package' + enterprise2_suffix(edition),
],
'commands': [
'ls dist/*.tar.gz*',
@@ -800,6 +795,13 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis
if archs:
cmd += ' -archs {}'.format(','.join(archs))
environment = {
'GCP_KEY': from_secret('gcp_key'),
}
if edition == 'enterprise2':
environment.update({'DOCKER_ENTERPRISE2_REPO': from_secret('docker_enterprise2_repo')})
return {
'name': 'build-docker-images' + ubuntu_sfx,
'image': 'google/cloud-sdk',
@@ -814,9 +816,7 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis
'name': 'docker',
'path': '/var/run/docker.sock'
}],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
},
'environment': environment
}
def fetch_images_step(edition):
@@ -827,6 +827,7 @@ def fetch_images_step(edition):
'GCP_KEY': from_secret('gcp_key'),
'DOCKER_USER': from_secret('docker_username'),
'DOCKER_PASSWORD': from_secret('docker_password'),
'DOCKER_ENTERPRISE2_REPO': from_secret('docker_enterprise2_repo'),
},
'commands': ['./bin/build artifacts docker fetch --edition {}'.format(edition)],
'depends_on': ['compile-build-cmd'],
@@ -838,13 +839,20 @@ def fetch_images_step(edition):
def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None):
name = docker_repo
docker_repo = 'grafana/{}'.format(docker_repo)
if mode == 'security':
mode = '--{} '.format(mode)
else:
mode = ''
cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {}'.format(
mode, docker_repo)
environment = {
'GCP_KEY': from_secret('gcp_key'),
'DOCKER_USER': from_secret('docker_username'),
'DOCKER_PASSWORD': from_secret('docker_password'),
}
cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {}'.format(mode, docker_repo)
if ver_mode == 'release':
deps = ['fetch-images-{}'.format(edition)]
@@ -852,14 +860,16 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None):
else:
deps = ['build-docker-images', 'build-docker-images-ubuntu']
if edition == 'enterprise2':
name = edition
docker_repo = '$${DOCKER_ENTERPRISE2_REPO}'
environment.update({'GCP_KEY': from_secret('gcp_key_hg'), 'DOCKER_ENTERPRISE2_REPO': from_secret('docker_enterprise2_repo')})
cmd = './bin/build artifacts docker publish-enterprise2 --dockerhub-repo {}'.format(docker_repo)
step = {
'name': 'publish-images-{}'.format(docker_repo),
'name': 'publish-images-{}'.format(name),
'image': 'google/cloud-sdk',
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'DOCKER_USER': from_secret('docker_username'),
'DOCKER_PASSWORD': from_secret('docker_password'),
},
'environment': environment,
'commands': [cmd],
'depends_on': deps,
'volumes': [{
scripts/drone/version.star
+5 -4
View File
@@ -1,11 +1,12 @@
load(
'scripts/drone/events/release.star',
'release_pipelines',
'oss_pipelines',
'enterprise_pipelines',
'enterprise2_pipelines',
)
ver_mode = 'release-branch'
trigger={'ref': ['refs/heads/v[0-9]*']}
def version_branch_pipelines():
return release_pipelines(ver_mode=ver_mode, trigger={
'ref': ['refs/heads/v[0-9]*'],
})
return oss_pipelines(ver_mode=ver_mode, trigger=trigger) + enterprise_pipelines(ver_mode=ver_mode, trigger=trigger) + enterprise2_pipelines(ver_mode=ver_mode, trigger=trigger)