public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: Remove initialize step (#47622) (#47770)

Browse Source 
* Remove initialize step from pr.star

* Remove initialize step from main.star

* Remove initialize step from release.star

* Reorder dependencies

* Re-add identify runner step

* Add enterprise steps

* Remove is_downstream variable from starlark logic (#47715)

* Make init steps depend on init-enterprise for enterprise pipelines

* Exclude shellcheck from enterprise pipelines

* Add init steps to enterprise integration tests pipeline

* Add dependency for windows enterprise step

* Update grabpl version

(cherry picked from commit 90892050d0)
This commit is contained in:
Dimitris Sotirakis
2022-04-14 15:36:16 +03:00
committed by GitHub
parent 5c41d84f88
commit ff6a7b0a7f
6 changed files with 680 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/drone/steps/lib.star
+166 -197
View File
@@ -1,6 +1,6 @@
load('scripts/drone/vault.star', 'from_secret', 'github_token', 'pull_secret', 'drone_token', 'prerelease_bucket')
grabpl_version = 'v2.9.33'
grabpl_version = 'v2.9.35'
build_image = 'grafana/build-container:1.5.3'
publish_image = 'grafana/grafana-ci-deploy:1.3.1'
deploy_docker_image = 'us.gcr.io/kubernetes-dev/drone/plugins/deploy-image'
@@ -18,6 +18,7 @@ trigger_oss = {
}
}
def slack_step(channel, template, secret):
return {
'name': 'slack',
@@ -30,106 +31,49 @@ def slack_step(channel, template, secret):
}
def initialize_step(edition, platform, ver_mode, is_downstream=False, install_deps=True):
if platform == 'windows':
return [
{
'name': 'identify-runner',
'image': windows_image,
'commands': [
'echo $env:DRONE_RUNNER_NAME',
],
},
]
common_cmds = [
# Generate Go code, will install Wire
# TODO: Install Wire in Docker image instead
'make gen-go',
]
def gen_version_step(ver_mode):
if ver_mode == 'release':
args = '${DRONE_TAG}'
common_cmds.append('./bin/grabpl verify-version ${DRONE_TAG}')
else:
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
build_no = '${DRONE_BUILD_NUMBER}'
args = '--build-id {}'.format(build_no)
identify_runner = identify_runner_step(platform)
if install_deps:
common_cmds.extend([
return {
'name': 'gen-version',
'image': build_image,
'depends_on': [
'grabpl',
],
'commands': [
'./bin/grabpl gen-version {}'.format(args),
],
}
def yarn_install_step():
return {
'name': 'yarn-install',
'image': build_image,
'commands': [
'yarn install --immutable',
])
if edition in ('enterprise', 'enterprise2'):
source_commit = ''
if ver_mode == 'release':
committish = '${DRONE_TAG}'
source_commit = ' ${DRONE_TAG}'
environment = {
'GITHUB_TOKEN': from_secret(github_token),
}
token = "--github-token $${GITHUB_TOKEN}"
elif ver_mode == 'release-branch':
committish = '${DRONE_BRANCH}'
environment = {}
token = ""
else:
environment = {}
if is_downstream:
source_commit = ' $${SOURCE_COMMIT}'
committish = '${DRONE_COMMIT}'
token = ""
steps = [
identify_runner,
clone_enterprise(committish),
{
'name': 'initialize',
'image': build_image,
'depends_on': [
'clone-enterprise',
],
'environment': environment,
'commands': [
'mv bin/grabpl /tmp/',
'rmdir bin',
'mv grafana-enterprise /tmp/',
'/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}'.format(token, source_commit),
'mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json',
'mkdir bin',
'mv /tmp/grabpl bin/'
] + common_cmds,
},
]
return steps
steps = [
identify_runner,
{
'name': 'initialize',
'image': build_image,
'commands': common_cmds,
},
]
return steps
],
'depends_on': [
'grabpl',
],
}
def identify_runner_step(platform):
if platform == 'linux':
return {
'name': 'identify-runner',
'image': alpine_image,
'commands': [
'echo $DRONE_RUNNER_NAME',
],
}
else:
def wire_install_step():
return {
'name': 'wire-install',
'image': build_image,
'commands': [
'make gen-go',
],
}
def identify_runner_step(platform='linux'):
if platform == 'windows':
return {
'name': 'identify-runner',
'image': windows_image,
@@ -137,9 +81,23 @@ def identify_runner_step(platform):
'echo $env:DRONE_RUNNER_NAME',
],
}
else:
return {
'name': 'identify-runner',
'image': alpine_image,
'commands': [
'echo $DRONE_RUNNER_NAME',
],
}
def clone_enterprise(committish):
def clone_enterprise_step(ver_mode):
if ver_mode == 'release':
committish = '${DRONE_TAG}'
elif ver_mode == 'release-branch':
committish = '${DRONE_BRANCH}'
else:
committish = '${DRONE_COMMIT}'
return {
'name': 'clone-enterprise',
'image': build_image,
@@ -153,8 +111,51 @@ def clone_enterprise(committish):
],
}
def init_enterprise_step(ver_mode):
source_commit = ''
if ver_mode == 'release':
source_commit = ' ${DRONE_TAG}'
environment = {
'GITHUB_TOKEN': from_secret(github_token),
}
token = "--github-token $${GITHUB_TOKEN}"
elif ver_mode == 'release-branch':
environment = {}
token = ""
else:
environment = {}
token = ""
return {
'name': 'init-enterprise',
'image': build_image,
'depends_on': [
'clone-enterprise',
],
'environment': environment,
'commands': [
'mv bin/grabpl /tmp/',
'rmdir bin',
'mv grafana-enterprise /tmp/',
'/tmp/grabpl init-enterprise {} /tmp/grafana-enterprise{}'.format(token, source_commit),
'mv /tmp/grafana-enterprise/deployment_tools_config.json deployment_tools_config.json',
'mkdir bin',
'mv /tmp/grabpl bin/'
],
}
def download_grabpl_step(platform="linux"):
if platform == 'windows':
return {
'name': 'grabpl',
'image': wix_image,
'commands': [
'$$ProgressPreference = "SilentlyContinue"',
'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/{}/windows/grabpl.exe -OutFile grabpl.exe'.format(
grabpl_version),
]
}
def download_grabpl_step():
return {
'name': 'grabpl',
'image': curl_image,
@@ -211,7 +212,7 @@ def lint_backend_step(edition):
'CGO_ENABLED': '1',
},
'depends_on': [
'initialize',
'wire-install',
],
'commands': [
# Don't use Make since it will re-download the linters
@@ -224,9 +225,6 @@ def benchmark_ldap_step():
return {
'name': 'benchmark-ldap',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'LDAP_HOSTNAME': 'ldap',
},
@@ -263,7 +261,6 @@ def store_storybook_step(edition, ver_mode, trigger=None):
if edition in ('enterprise', 'enterprise2'):
return None
commands = []
if ver_mode == 'release':
commands.extend([
@@ -273,12 +270,12 @@ def store_storybook_step(edition, ver_mode, trigger=None):
else:
# main pipelines should deploy storybook to grafana-storybook/canary public bucket
commands = ['./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook',]
commands = ['./bin/grabpl store-storybook --deployment canary --src-bucket grafana-storybook', ]
step = {
'name': 'store-storybook',
'image': publish_image,
'depends_on': ['build-storybook',] + end_to_end_tests_deps(edition),
'depends_on': ['build-storybook', ] + end_to_end_tests_deps(edition),
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket)
@@ -289,6 +286,7 @@ def store_storybook_step(edition, ver_mode, trigger=None):
step.update(trigger)
return step
def e2e_tests_artifacts(edition):
return {
'name': 'e2e-tests-artifacts-upload' + enterprise2_suffix(edition),
@@ -363,7 +361,7 @@ def upload_cdn_step(edition, ver_mode, trigger=None):
return step
def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
def build_backend_step(edition, ver_mode, variants=None):
variants_str = ''
if variants:
variants_str = ' --variants {}'.format(','.join(variants))
@@ -379,10 +377,7 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
),
]
else:
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
build_no = '${DRONE_BUILD_NUMBER}'
env = {}
cmds = [
'./bin/grabpl build-backend --jobs 8 --edition {} --build-id {}{}'.format(
@@ -393,19 +388,17 @@ def build_backend_step(edition, ver_mode, variants=None, is_downstream=False):
return {
'name': 'build-backend' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'initialize',
],
'environment': env,
'depends_on': [
'gen-version',
'wire-install',
],
'commands': cmds,
}
def build_frontend_step(edition, ver_mode, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
def build_frontend_step(edition, ver_mode):
build_no = '${DRONE_BUILD_NUMBER}'
# TODO: Use percentage for num jobs
if ver_mode == 'release':
@@ -422,42 +415,42 @@ def build_frontend_step(edition, ver_mode, is_downstream=False):
return {
'name': 'build-frontend',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'NODE_OPTIONS': '--max_old_space_size=8192',
},
'depends_on': [
'gen-version',
'yarn-install',
],
'commands': cmds,
}
def build_frontend_package_step(edition, ver_mode, is_downstream=False):
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
def build_frontend_package_step(edition, ver_mode):
build_no = '${DRONE_BUILD_NUMBER}'
# TODO: Use percentage for num jobs
if ver_mode == 'release':
cmds = [
'./bin/grabpl build-frontend-packages --jobs 8 --github-token $${GITHUB_TOKEN} ' + \
'--edition {} ${{DRONE_TAG}}'.format(edition),
]
]
else:
cmds = [
'./bin/grabpl build-frontend-packages --jobs 8 --edition {} '.format(edition) + \
'--build-id {}'.format(build_no),
]
]
return {
'name': 'build-frontend-packages',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'NODE_OPTIONS': '--max_old_space_size=8192',
},
'depends_on': [
'gen-version',
'yarn-install',
],
'commands': cmds,
}
@@ -487,10 +480,11 @@ def build_plugins_step(edition, sign=False):
return {
'name': 'build-plugins',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': env,
'depends_on': [
'gen-version',
'yarn-install',
],
'commands': [
# TODO: Use percentage for num jobs
'./bin/grabpl build-plugins --jobs 8 --edition {}{}'.format(edition, sign_args),
@@ -503,7 +497,7 @@ def test_backend_step(edition):
'name': 'test-backend' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'initialize',
'wire-install',
],
'commands': [
'./bin/grabpl test-backend --edition {}'.format(edition),
@@ -516,7 +510,7 @@ def test_backend_integration_step(edition):
'name': 'test-backend-integration' + enterprise2_suffix(edition),
'image': build_image,
'depends_on': [
'initialize',
'wire-install',
],
'commands': [
'./bin/grabpl integration-tests --edition {}'.format(edition),
@@ -528,12 +522,12 @@ def test_frontend_step():
return {
'name': 'test-frontend',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'TEST_MAX_WORKERS': '50%',
},
'depends_on': [
'yarn-install',
],
'commands': [
'yarn run ci:test-frontend',
],
@@ -544,16 +538,16 @@ def lint_frontend_step():
return {
'name': 'lint-frontend',
'image': build_image,
'depends_on': [
'initialize',
],
'environment': {
'TEST_MAX_WORKERS': '50%',
},
'depends_on': [
'yarn-install',
],
'commands': [
'yarn run prettier:check',
'yarn run lint',
'yarn run i18n:compile', # TODO: right place for this?
'yarn run i18n:compile', # TODO: right place for this?
'yarn run typecheck',
],
}
@@ -617,9 +611,6 @@ def codespell_step():
return {
'name': 'codespell',
'image': build_image,
'depends_on': [
'initialize',
],
'commands': [
# Important: all words have to be in lowercase, and separated by "\n".
'echo -e "unknwon\nreferer\nerrorstring\neror\niam\nwan" > words_to_ignore.txt',
@@ -634,7 +625,7 @@ def shellcheck_step():
'name': 'shellcheck',
'image': build_image,
'depends_on': [
'initialize',
'grabpl',
],
'commands': [
'./bin/grabpl shellcheck',
@@ -642,7 +633,7 @@ def shellcheck_step():
}
def package_step(edition, ver_mode, include_enterprise2=False, variants=None, is_downstream=False):
def package_step(edition, ver_mode, include_enterprise2=False, variants=None):
deps = [
'build-plugins',
'build-backend',
@@ -683,10 +674,7 @@ def package_step(edition, ver_mode, include_enterprise2=False, variants=None, is
),
]
else:
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
build_no = '${DRONE_BUILD_NUMBER}'
cmds = [
'{}./bin/grabpl package --jobs 8 --edition {} '.format(test_args, edition) + \
'--build-id {}{}{}'.format(build_no, variants_str, sign_args),
@@ -731,6 +719,7 @@ def grafana_server_step(edition, port=3001):
],
}
def e2e_tests_step(suite, edition, port=3001, tries=None):
cmd = './bin/grabpl e2e-tests --port {} --suite {}'.format(port, suite)
if tries:
@@ -810,13 +799,15 @@ def build_docker_images_step(edition, ver_mode, archs=None, ubuntu=False, publis
},
}
def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None):
if mode == 'security':
mode = '--{} '.format(mode)
else:
mode = ''
cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(mode, docker_repo)
cmd = './bin/grabpl artifacts docker publish {}--dockerhub-repo {} --base alpine --base ubuntu --arch amd64 --arch arm64 --arch armv7'.format(
mode, docker_repo)
if ver_mode == 'release':
deps = ['fetch-images-{}'.format(edition)]
@@ -847,10 +838,7 @@ def publish_images_step(edition, ver_mode, mode, docker_repo, trigger=None):
def postgres_integration_tests_step(edition, ver_mode):
deps = []
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'):
deps.extend(['initialize'])
else:
deps.extend(['grabpl'])
deps.extend(['grabpl'])
return {
'name': 'postgres-integration-tests',
'image': build_image,
@@ -875,10 +863,7 @@ def postgres_integration_tests_step(edition, ver_mode):
def mysql_integration_tests_step(edition, ver_mode):
deps = []
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'):
deps.extend(['initialize'])
else:
deps.extend(['grabpl'])
deps.extend(['grabpl'])
return {
'name': 'mysql-integration-tests',
'image': build_image,
@@ -901,10 +886,7 @@ def mysql_integration_tests_step(edition, ver_mode):
def redis_integration_tests_step(edition, ver_mode):
deps = []
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'):
deps.extend(['initialize'])
else:
deps.extend(['grabpl'])
deps.extend(['grabpl'])
return {
'name': 'redis-integration-tests',
'image': build_image,
@@ -921,10 +903,7 @@ def redis_integration_tests_step(edition, ver_mode):
def memcached_integration_tests_step(edition, ver_mode):
deps = []
if edition in ('enterprise', 'enterprise2') and ver_mode in ('release-branch', 'release'):
deps.extend(['initialize'])
else:
deps.extend(['grabpl'])
deps.extend(['grabpl'])
return {
'name': 'memcached-integration-tests',
'image': build_image,
@@ -965,15 +944,16 @@ def enterprise2_suffix(edition):
return ''
def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None):
if ver_mode == 'main' and edition in ('enterprise', 'enterprise2') and not is_downstream:
def upload_packages_step(edition, ver_mode, trigger=None):
if ver_mode == 'main' and edition in ('enterprise', 'enterprise2'):
return None
if ver_mode == 'release':
packages_bucket = '$${{PRERELEASE_BUCKET}}/artifacts/downloads{}'.format(enterprise2_suffix(edition))
cmd = './bin/grabpl upload-packages --edition {} --packages-bucket {}'.format(edition, packages_bucket)
elif edition == 'enterprise2':
cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads-enterprise2'.format(edition)
cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads-enterprise2'.format(
edition)
else:
cmd = './bin/grabpl upload-packages --edition {} --packages-bucket grafana-downloads'.format(edition)
@@ -981,7 +961,7 @@ def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None):
if edition in 'enterprise2' or not end_to_end_tests_deps(edition):
deps.extend([
'package' + enterprise2_suffix(edition),
])
])
else:
deps.extend(end_to_end_tests_deps(edition))
@@ -1000,16 +980,13 @@ def upload_packages_step(edition, ver_mode, is_downstream=False, trigger=None):
return step
def store_packages_step(edition, ver_mode, is_downstream=False):
def store_packages_step(edition, ver_mode):
if ver_mode == 'release':
cmd = './bin/grabpl store-packages --edition {} --packages-bucket grafana-downloads --gcp-key /tmp/gcpkey.json ${{DRONE_TAG}}'.format(
edition,
)
elif ver_mode == 'main':
if not is_downstream:
build_no = '${DRONE_BUILD_NUMBER}'
else:
build_no = '$${SOURCE_BUILD_NUMBER}'
build_no = '${DRONE_BUILD_NUMBER}'
cmd = './bin/grabpl store-packages --edition {} --gcp-key /tmp/gcpkey.json --build-id {}'.format(
edition, build_no,
)
@@ -1035,12 +1012,7 @@ def store_packages_step(edition, ver_mode, is_downstream=False):
}
def get_windows_steps(edition, ver_mode, is_downstream=False):
if not is_downstream:
source_commit = ''
else:
source_commit = ' $$env:SOURCE_COMMIT'
def get_windows_steps(edition, ver_mode):
init_cmds = []
sfx = ''
if edition in ('enterprise', 'enterprise2'):
@@ -1053,12 +1025,12 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
])
steps = [
{
'name': 'initialize',
'name': 'windows-init',
'image': wix_image,
'commands': init_cmds,
},
]
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2') or is_downstream)) or ver_mode in (
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
'release', 'release-branch',
):
bucket_part = ''
@@ -1070,10 +1042,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
dir = 'main'
bucket = 'grafana-downloads'
bucket_part = ' --packages-bucket {}'.format(bucket)
if not is_downstream:
build_no = 'DRONE_BUILD_NUMBER'
else:
build_no = 'SOURCE_BUILD_NUMBER'
build_no = 'DRONE_BUILD_NUMBER'
ver_part = '--build-id $$env:{}'.format(build_no)
installer_commands = [
'$$gcpKey = $$env:GCP_KEY',
@@ -1084,7 +1053,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'rm gcpkey.json',
'cp C:\\App\\nssm-2.24.zip .',
]
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2') or is_downstream)) or ver_mode in (
if (ver_mode == 'main' and (edition not in ('enterprise', 'enterprise2'))) or ver_mode in (
'release',
):
installer_commands.extend([
@@ -1105,15 +1074,15 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
steps.append({
'name': 'build-windows-installer',
'image': wix_image,
'depends_on': [
'windows-init',
],
'environment': {
'GCP_KEY': from_secret('gcp_key'),
'PRERELEASE_BUCKET': from_secret(prerelease_bucket),
'GITHUB_TOKEN': from_secret('github_token')
},
'commands': installer_commands,
'depends_on': [
'initialize',
],
})
if edition in ('enterprise', 'enterprise2'):
@@ -1132,11 +1101,10 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
clone_cmds = [
'git clone "https://$$env:GITHUB_TOKEN@github.com/grafana/grafana-enterprise.git"',
]
if not is_downstream:
clone_cmds.extend([
'cd grafana-enterprise',
'git checkout {}'.format(committish),
])
clone_cmds.extend([
'cd grafana-enterprise',
'git checkout {}'.format(committish),
])
steps.insert(0, {
'name': 'clone',
'image': wix_image,
@@ -1154,7 +1122,7 @@ def get_windows_steps(edition, ver_mode, is_downstream=False):
'rm -r -force grafana-enterprise',
'cp grabpl.exe C:\\App\\grabpl.exe',
'rm -force grabpl.exe',
'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise{}'.format(source_commit),
'C:\\App\\grabpl.exe init-enterprise --github-token $$env:GITHUB_TOKEN C:\\App\\grafana-enterprise',
'cp C:\\App\\grabpl.exe grabpl.exe',
])
if 'environment' in steps[1]:
@@ -1194,6 +1162,7 @@ def ensure_cuetsified_step():
],
}
def end_to_end_tests_deps(edition):
if disable_tests:
return []