diff --git a/.drone.yml b/.drone.yml index 19ae07730d5..12c44b520d1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -571,6 +571,8 @@ steps: name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - yarn packages:pack + - ./scripts/validate-npm-packages.sh depends_on: - compile-build-cmd - yarn-install @@ -1458,6 +1460,8 @@ steps: name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - yarn packages:pack + - ./scripts/validate-npm-packages.sh depends_on: - compile-build-cmd - yarn-install @@ -1734,7 +1738,7 @@ steps: repo: - grafana/grafana - commands: - - ./scripts/circle-release-canary-packages.sh + - ./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org/' depends_on: - end-to-end-tests-dashboards-suite - end-to-end-tests-panels-suite @@ -1746,6 +1750,9 @@ steps: image: grafana/build-container:1.7.3 name: release-canary-npm-packages when: + paths: + include: + - packages/** repo: - grafana/grafana - commands: @@ -4685,6 +4692,8 @@ steps: name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition oss --build-id ${DRONE_BUILD_NUMBER} + - yarn packages:pack + - ./scripts/validate-npm-packages.sh depends_on: - compile-build-cmd - yarn-install @@ -5328,6 +5337,8 @@ steps: name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} + - yarn packages:pack + - ./scripts/validate-npm-packages.sh depends_on: - compile-build-cmd - yarn-install @@ -6049,6 +6060,8 @@ steps: name: build-frontend - commands: - ./bin/build build-frontend-packages --jobs 8 --edition enterprise --build-id ${DRONE_BUILD_NUMBER} + - yarn packages:pack + - ./scripts/validate-npm-packages.sh depends_on: - compile-build-cmd - yarn-install @@ -6810,6 +6823,6 @@ kind: secret name: enterprise2_security_prefix --- kind: signature -hmac: d3567c954d0ac9f1f47d55bc85eb6594379c888fe591428f2dcd0146f2bd3ca2 +hmac: 7f282aaf4f3ba496f2d954241a16afd2facbf2cf1ef04dfb75cfec1b42fd39d1 ... diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 07c06825e28..2d551c7a4df 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -455,6 +455,8 @@ lerna.json @grafana/frontend-ops /scripts/check-breaking-changes.sh @grafana/plugins-platform-frontend /scripts/ci-* @grafana/grafana-delivery /scripts/circle-* @grafana/grafana-delivery +/scripts/publish-npm-packages.sh @grafana/grafana-delivery @grafana/plugins-platform-frontend +/scripts/validate-npm-packages.sh @grafana/grafana-delivery @grafana/plugins-platform-frontend /scripts/ci-frontend-metrics.sh @grafana/grafana-frontend-platform @grafana/plugins-platform-frontend @grafana/grafana-bi-squad /scripts/cli/ @grafana/grafana-frontend-platform /scripts/clean-git-or-error.sh @grafana/grafana-as-code diff --git a/.gitignore b/.gitignore index 39b7a1e2ae4..fbfb6a6396f 100644 --- a/.gitignore +++ b/.gitignore @@ -139,7 +139,7 @@ pkg/services/quota/quotaimpl/storage/storage.json /packages/**/package.tgz /packages/grafana-toolkit/sass ## CI places the packages in a different location -/npm-artifacts/*.tgz +/npm-artifacts # Ignore frontend build manifest manifest.json diff --git a/devenv/local-npm/conf/config.yaml b/devenv/local-npm/conf/config.yaml new file mode 100644 index 00000000000..26686016f46 --- /dev/null +++ b/devenv/local-npm/conf/config.yaml @@ -0,0 +1,202 @@ +# +# This is the default configuration file. It allows all users to do anything, +# please read carefully the documentation and best practices to +# improve security. +# +# Do not configure host and port under `listen` in this file +# as it will be ignored when using docker. +# see https://verdaccio.org/docs/en/docker#docker-and-custom-port-configuration +# +# Look here for more config file examples: +# https://github.com/verdaccio/verdaccio/tree/5.x/conf +# +# Read about the best practices +# https://verdaccio.org/docs/best + +# path to a directory with all packages +storage: /verdaccio/storage/data +# path to a directory with plugins to include +plugins: /verdaccio/plugins + +# https://verdaccio.org/docs/webui +web: + title: Verdaccio + # comment out to disable gravatar support + # gravatar: false + # by default packages are ordercer ascendant (asc|desc) + # sort_packages: asc + # convert your UI to the dark side + # darkMode: true + # html_cache: true + # by default all features are displayed + # login: true + # showInfo: true + # showSettings: true + # In combination with darkMode you can force specific theme + # showThemeSwitch: true + # showFooter: true + # showSearch: true + # showRaw: true + # showDownloadTarball: true + # HTML tags injected after manifest + # scriptsBodyAfter: + # - '' + # HTML tags injected before ends + # metaScripts: + # - '' + # - '' + # - '' + # HTML tags injected first child at + # bodyBefore: + # - '
html before webpack scripts
' + # Public path for template manifest scripts (only manifest) + # publicPath: http://somedomain.org/ + +# https://verdaccio.org/docs/configuration#authentication +auth: + htpasswd: + file: /verdaccio/storage/htpasswd + # Maximum amount of users allowed to register, defaults to "+infinity". + # You can set this to -1 to disable registration. + # max_users: 1000 + # Hash algorithm, possible options are: "bcrypt", "md5", "sha1", "crypt". + # algorithm: bcrypt # by default is crypt, but is recommended use bcrypt for new installations + # Rounds number for "bcrypt", will be ignored for other algorithms. + # rounds: 10 + +# https://verdaccio.org/docs/configuration#uplinks +# a list of other known repositories we can talk to +uplinks: + npmjs: + url: https://registry.npmjs.org/ + +# Learn how to protect your packages +# https://verdaccio.org/docs/protect-your-dependencies/ +# https://verdaccio.org/docs/configuration#packages +packages: + '@*/*': + # allow all users (including non-authenticated users) to read and + # publish scoped packages + access: $anonymous + publish: $anonymous + unpublish: $anonymous + proxy: npmjs + + '**': + # allow all users (including non-authenticated users) to read and + # publish all packages + # + # you can specify usernames/groupnames (depending on your auth plugin) + # and three keywords: "$all", "$anonymous", "$authenticated" + access: $anonymous + + # allow all known users to publish/publish packages + # (anyone can register by default, remember?) + publish: $anonymous + unpublish: $anonymous + + # if package is not available locally, proxy requests to 'npmjs' registry + proxy: npmjs + +# To improve your security configuration and avoid dependency confusion +# consider removing the proxy property for private packages +# https://verdaccio.org/docs/best#remove-proxy-to-increase-security-at-private-packages + +# https://verdaccio.org/docs/configuration#server +# You can specify HTTP/1.1 server keep alive timeout in seconds for incoming connections. +# A value of 0 makes the http server behave similarly to Node.js versions prior to 8.0.0, which did not have a keep-alive timeout. +# WORKAROUND: Through given configuration you can workaround following issue https://github.com/verdaccio/verdaccio/issues/301. Set to 0 in case 60 is not enough. +server: + keepAliveTimeout: 60 + # Allow `req.ip` to resolve properly when Verdaccio is behind a proxy or load-balancer + # See: https://expressjs.com/en/guide/behind-proxies.html + # trustProxy: '127.0.0.1' + +# https://verdaccio.org/docs/configuration#offline-publish +# publish: +# allow_offline: false + +# https://verdaccio.org/docs/configuration#url-prefix +# url_prefix: /verdaccio/ +# VERDACCIO_PUBLIC_URL='https://somedomain.org'; +# url_prefix: '/my_prefix' +# // url -> https://somedomain.org/my_prefix/ +# VERDACCIO_PUBLIC_URL='https://somedomain.org'; +# url_prefix: '/' +# // url -> https://somedomain.org/ +# VERDACCIO_PUBLIC_URL='https://somedomain.org/first_prefix'; +# url_prefix: '/second_prefix' +# // url -> https://somedomain.org/second_prefix/' + +# https://verdaccio.org/docs/configuration#security +# security: +# api: +# legacy: true +# jwt: +# sign: +# expiresIn: 29d +# verify: +# someProp: [value] +# web: +# sign: +# expiresIn: 1h # 1 hour by default +# verify: +# someProp: [value] + +# https://verdaccio.org/docs/configuration#user-rate-limit +# userRateLimit: +# windowMs: 50000 +# max: 1000 + +# https://verdaccio.org/docs/configuration#max-body-size +# max_body_size: 10mb + +# https://verdaccio.org/docs/configuration#listen-port +# listen: +# - localhost:4873 # default value +# - http://localhost:4873 # same thing +# - 0.0.0.0:4873 # listen on all addresses (INADDR_ANY) +# - https://example.org:4873 # if you want to use https +# - "[::1]:4873" # ipv6 +# - unix:/tmp/verdaccio.sock # unix socket + +# The HTTPS configuration is useful if you do not consider use a HTTP Proxy +# https://verdaccio.org/docs/configuration#https +# https: +# key: ./path/verdaccio-key.pem +# cert: ./path/verdaccio-cert.pem +# ca: ./path/verdaccio-csr.pem + +# https://verdaccio.org/docs/configuration#proxy +# http_proxy: http://something.local/ +# https_proxy: https://something.local/ + +# https://verdaccio.org/docs/configuration#notifications +# notify: +# method: POST +# headers: [{ "Content-Type": "application/json" }] +# endpoint: https://usagge.hipchat.com/v2/room/3729485/notification?auth_token=mySecretToken +# content: '{"color":"green","message":"New package published: * {{ name }}*","notify":true,"message_format":"text"}' + +middlewares: + audit: + enabled: true + +# https://verdaccio.org/docs/logger +# log settings +logs: { type: stdout, format: pretty, level: http } +#experiments: +# # support for npm token command +# token: false +# # enable tarball URL redirect for hosting tarball with a different server, the tarball_url_redirect can be a template string +# tarball_url_redirect: 'https://mycdn.com/verdaccio/${packageName}/${filename}' +# # the tarball_url_redirect can be a function, takes packageName and filename and returns the url, when working with a js configuration file +# tarball_url_redirect(packageName, filename) { +# const signedUrl = // generate a signed url +# return signedUrl; +# } + +# translate your registry, api i18n not available yet +# i18n: +# list of the available translations https://github.com/verdaccio/verdaccio/blob/master/packages/plugins/ui-theme/src/i18n/ABOUT_TRANSLATIONS.md +# web: en-US \ No newline at end of file diff --git a/devenv/local-npm/docker-compose.yaml b/devenv/local-npm/docker-compose.yaml index 1a0acdc8a25..be0ea5220ec 100644 --- a/devenv/local-npm/docker-compose.yaml +++ b/devenv/local-npm/docker-compose.yaml @@ -7,7 +7,7 @@ services: ports: - "4873:4873" volumes: - - verdaccio:/verdaccio + - ./conf:/verdaccio/conf volumes: verdaccio: diff --git a/package.json b/package.json index b525589d2e0..0e3fbdef90e 100644 --- a/package.json +++ b/package.json @@ -29,12 +29,6 @@ "packages:clean": "rimraf ./npm-artifacts && lerna run clean --parallel", "packages:prepare": "lerna version --no-push --no-git-tag-version --force-publish --exact", "packages:pack": "mkdir -p ./npm-artifacts && lerna exec --no-private -- yarn pack --out \"../../npm-artifacts/%s-%v.tgz\"", - "packages:publish": "lerna exec --no-private -- npm publish package.tgz", - "packages:publishCanary": "lerna exec --no-private -- npm publish package.tgz --tag canary", - "packages:publishLatest": "lerna exec --no-private -- npm publish package.tgz", - "packages:publishNext": "lerna exec --no-private -- npm publish package.tgz --tag next", - "packages:publishTest": "lerna exec --no-private -- npm publish package.tgz --tag test", - "packages:publishDev": "lerna exec --no-private -- npm publish package.tgz --tag dev --registry http://localhost:4873", "packages:typecheck": "lerna run typecheck", "precommit": "yarn run lint-staged", "prettier:check": "prettier --check --list-different=false --loglevel=warn \"**/*.{ts,tsx,scss,md,mdx}\"", diff --git a/packages/README.md b/packages/README.md index 4fe45149509..6fc9c3cd7b7 100644 --- a/packages/README.md +++ b/packages/README.md @@ -36,20 +36,21 @@ Every commit to main that has changes within the `packages` directory is a subje > All of the steps below must be performed on a release branch, according to Grafana Release Guide. -> Make sure you are logged in to npm in your terminal and that you are a part of Grafana org on npm. +> You must be logged in to NPM as part of Grafana NPM org before attempting to publish to the npm registery. -1. Run `yarn packages:prepare` script from the root directory. This performs tests on the packages and prompts for the version of the packages. The version should be the same as the one being released. +1. Run `yarn packages:clean` script from the root directory. This will delete any previous builds of the packages. +2. Run `yarn packages:prepare` script from the root directory. This performs tests on the packages and prompts for the version of the packages. The version should be the same as the one being released. - Make sure you use semver convention. So, _place a dot between prerelease id and prerelease number_, i.e. 6.3.0-alpha.1 - Make sure you confirm the version bump when prompted! -2. Run `yarn packages:build` script that compiles distribution code in `packages/grafana-*/dist`. -3. Run `yarn packages:pack` script to zip each package into `.tgz`. This is required for yarn berry to replace properties in the package.json files declared in `publishConfig`. -4. Depending whether or not it's a prerelease: +3. Run `yarn packages:build` script that compiles distribution code in `packages/grafana-*/dist`. +4. Run `yarn packages:pack` script to compress each package into `npm-artifacts/*.tgz` files. This is required for yarn to replace properties in the package.json files declared in the `publishConfig` property. +5. Depending on whether or not it's a prerelease: - - When releasing a prerelease run `packages:publishNext` to publish new versions. - - When releasing a stable version run `packages:publishLatest` to publish new versions. - - When releasing a test version run `packages:publishTest` to publish test versions. + - When releasing a prerelease run `./scripts/publish-npm-packages.sh --dist-tag 'next' --registry 'https://registry.npmjs.org/'` to publish new versions. + - When releasing a stable version run `./scripts/publish-npm-packages.sh --dist-tag 'latest' --registry 'https://registry.npmjs.org/'` to publish new versions. + - When releasing a test version run `./scripts/publish-npm-packages.sh --dist-tag 'test' --registry 'https://registry.npmjs.org/'` to publish test versions. -5. Push version commit to the release branch. +6. Revert any changes made by the `packages:prepare` script. ### Building individual packages @@ -72,21 +73,21 @@ In this guide you will set up [Verdaccio](https://verdaccio.org/) registry local From your terminal: 1. Navigate to `devenv/local-npm` directory. -2. Run `docker-compose up`. This will start your local npm registry, available at http://localhost:4873/ -3. Run `npm login --registry=http://localhost:4873 --scope=@grafana` . This will allow you to publish any @grafana/\* package into the local registry. -4. Run `npm config set @grafana:registry http://localhost:4873`. This will config your npm to install @grafana scoped packages from your local registry. +2. Run `docker-compose up`. This will start your local npm registry, available at http://localhost:4873/. Note the verdaccio config allows +3. To test `@grafana` packages published to your local npm registry uncomment `npmScopes` and `unsafeHttpWhitelist` properties in the `.yarnrc` file. #### Publishing packages to local npm registry -You need to follow [manual packages release procedure](#manual-release). The only difference is you need to run `yarn packages:publishDev` task in order to publish to you local registry. +You need to follow [manual packages release procedure](#manual-release). The only difference is the last command in order to publish to you local registry. From your terminal: -1. Run `yarn packages:prepare`. -2. Run `yarn packages:build`. -3. Run `yarn packages:pack`. -4. Run `yarn packages:publishDev`. -5. Navigate to http://localhost:4873 and verify that version was published +1. Run `yarn packages:clean`. +2. Run `yarn packages:prepare`. +3. Run `yarn packages:build`. +4. Run `yarn packages:pack`. +5. Run `./scripts/publish-npm-packages.sh`. +6. Navigate to http://localhost:4873 and verify the version was published Locally published packages will be published under `dev` channel, so in your plugin package.json file you can use that channel. For example: diff --git a/scripts/circle-release-canary-packages.sh b/scripts/circle-release-canary-packages.sh deleted file mode 100755 index 4487282a782..00000000000 --- a/scripts/circle-release-canary-packages.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# shellcheck source=./scripts/helpers/exit-if-fail.sh -source "$(dirname "$0")/helpers/exit-if-fail.sh" - -# check if there were any changes to packages between current and previous commit -count=$(git diff HEAD~1..HEAD --name-only -- packages | awk '{c++} END {print c}') - -if [ -z "$count" ]; then - echo "No changes in packages, skipping packages publishing" -else - echo "Changes detected in ${count} packages" - echo "Starting to release latest canary version" - - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> ~/.npmrc - - echo $'\nPacking packages' - yarn packages:pack - - echo $'\nPublishing packages' - for file in ./npm-artifacts/*.tgz; do npm publish "$file" --tag canary; done - -fi diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index f6d448027f7..3cba0e8da70 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -567,6 +567,8 @@ def build_frontend_package_step(edition, ver_mode): cmds = [ "./bin/build build-frontend-packages --jobs 8 --edition {} ".format(edition) + "--build-id {}".format(build_no), + "yarn packages:pack", + "./scripts/validate-npm-packages.sh", ] return { @@ -1179,11 +1181,21 @@ def release_canary_npm_packages_step(trigger = None): "NPM_TOKEN": from_secret("npm_token"), }, "commands": [ - "./scripts/circle-release-canary-packages.sh", + "./scripts/publish-npm-packages.sh --dist-tag 'canary' --registry 'https://registry.npmjs.org/'", ], } if trigger: - step = dict(step, when = trigger) + step = dict( + step, + when = dict( + trigger, + paths = { + "include": [ + "packages/**", + ], + }, + ), + ) return step def enterprise2_suffix(edition): diff --git a/scripts/publish-npm-packages.sh b/scripts/publish-npm-packages.sh new file mode 100755 index 00000000000..2437403626d --- /dev/null +++ b/scripts/publish-npm-packages.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Set default values for dist-tag and registry for local development +# to prevent running this script and accidentally publishing to npm +dist_tag="canary" +registry="http://localhost:4873" + +# shellcheck source=./scripts/helpers/exit-if-fail.sh +source "$(dirname "$0")/helpers/exit-if-fail.sh" + +if [ -z "$NPM_TOKEN" ]; then + echo "The NPM_TOKEN environment variable does not exist." + exit 1 +fi + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + --dist-tag) + dist_tag="$2" + shift # past argument + shift # past value + ;; + --registry) + registry="$2" + shift # past argument + shift # past value + ;; + *) # unknown option + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +echo "Starting to release $dist_tag version" + +echo "$registry/:_authToken=${NPM_TOKEN}" >> ~/.npmrc + +# Loop over .tar files in directory and publish them to npm registry +for file in ./npm-artifacts/*.tgz; do + npm publish "$file" --tag "$dist_tag" --registry "$registry" +done diff --git a/scripts/validate-npm-packages.sh b/scripts/validate-npm-packages.sh new file mode 100755 index 00000000000..a07ebe5a662 --- /dev/null +++ b/scripts/validate-npm-packages.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# This script is used to validate the npm packages that are published to npmjs.org are in the correct format. +# It won't catch things like malformed JS or Types but it will assert that the package has +# the correct files and package.json properties. +ARTIFACTS_DIR="./npm-artifacts" + +for file in "$ARTIFACTS_DIR"/*.tgz; do + echo "🔍 Checking NPM package: $file" + # get filename then strip everything after package name. + dir_name=$(basename "$file" .tgz | sed 's/^@\(.*\)-[0-9]*[.]*[0-9]*[.]*[0-9]*-\([0-9]*[a-zA-Z]*\)/\1/') + mkdir -p "./npm-artifacts/$dir_name" + tar -xzf "$file" -C "./npm-artifacts/$dir_name" --strip-components=1 + + # Make sure the tar wasn't empty + if [ ! -d "./npm-artifacts/$dir_name" ]; then + echo -e "❌ Failed: Empty package $dir_name.\n" + exit 1 + fi + + # Navigate inside the new extracted directory + pushd "./npm-artifacts/$dir_name" || exit + + # Check for required files + check_files=("package.json" "README.md" "CHANGELOG.md" "LICENSE_APACHE2") + for check_file in "${check_files[@]}"; do + if [ ! -f "$check_file" ]; then + echo -e "❌ Failed: Missing required file $check_file in package $dir_name.\n" + exit 1 + fi + done + + # @grafana/toolkit structure is different to the other packages + if [[ "$dir_name" == "grafana-toolkit" ]]; then + if [ ! -d bin ] || [ ! -f bin/grafana-toolkit.js ]; then + echo -e "❌ Failed: Missing 'bin' directory or required files in package $dir_name.\n" + exit 1 + fi + + echo -e "✅ Passed: package checks for $file.\n" + popd || exit + continue + fi + + # Assert commonjs builds + if [ ! -d dist ] || [ ! -f dist/index.js ] || [ ! -f dist/index.d.ts ]; then + echo -e "❌ Failed: Missing 'dist' directory or required commonjs files in package $dir_name.\n" + exit 1 + fi + + if [ "$(jq -r '.main' package.json)" != "dist/index.js" ] || \ + [ "$(jq -r '.types' package.json)" != "dist/index.d.ts" ]; then + echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n" + exit 1 + fi + + # Assert esm builds + esm_packages=("grafana-data" "grafana-ui" "grafana-runtime" "grafana-e2e-selectors" "grafana-schema") + for esm_package in "${esm_packages[@]}"; do + if [[ "$dir_name" == "$esm_package" ]]; then + if [ ! -d dist/esm ] || [ ! -f dist/esm/index.js ]; then + echo -e "❌ Failed: Missing 'dist/esm' directory or required esm files in package $dir_name.\n" + exit 1 + fi + + if [ "$(jq -r '.module' package.json)" != "dist/esm/index.js" ]; then + echo -e "❌ Failed: Incorrect package.json properties in package $dir_name.\n" + exit 1 + fi + fi + done + + echo -e "✅ Passed: package checks for $file.\n" + popd || exit + +done + +echo "🚀 All NPM package checks passed! 🚀" +rm -rf "${ARTIFACTS_DIR:?}/"*/ +exit 0