FS: Speed up backend build-reload cycle (#108889)
* wip so far. need to prove rebuilds work * build common runner image * glibc support * ubuntu * add conditional * add support for cross-os builds * format * don't use zig by default on macos * use postgres * tidy up * more tidy * update readme * remove errant changes
This commit is contained in:
Josh Hunt
@@ -0,0 +1 @@
|
||||
build
|
||||
@@ -25,4 +25,4 @@ To simulate the `/bootdata` endpoint being available, there are special control
|
||||
- `/-/down/:seconds` - Simulates the endpoint being unavailable for a custom number of seconds.
|
||||
- `/-/up` - Restores the endpoint to being available.
|
||||
|
||||
When unavailable, the API will return `HTTP 503 Service Unavailable` with a JSON payload.
|
||||
When unavailable, the API will return `HTTP 503 Service Unavailable` with a JSON payload.
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
config.define_bool("use-zig")
|
||||
cfg = config.parse()
|
||||
|
||||
use_zig = cfg.get('docker-builder', False)
|
||||
|
||||
# --- Frontend processes
|
||||
local_resource(
|
||||
'yarn install',
|
||||
@@ -5,7 +10,7 @@ local_resource(
|
||||
deps=[
|
||||
'yarn.lock',
|
||||
],
|
||||
labels=["frontend"]
|
||||
labels=["local"]
|
||||
)
|
||||
|
||||
local_resource(
|
||||
@@ -13,63 +18,94 @@ local_resource(
|
||||
cmd='rm -rf public/build/assets-manifest.json',
|
||||
serve_cmd='yarn start:noLint',
|
||||
resource_deps=['yarn install'],
|
||||
|
||||
# Note: this doesn't seem to work as expected - the assets-manifest is somehow created before
|
||||
# the webpack build is complete?
|
||||
readiness_probe=probe(
|
||||
initial_delay_secs=5, # wait for the assets-manifest.json to first be deleted
|
||||
initial_delay_secs=10,
|
||||
period_secs=1,
|
||||
exec=exec_action(["bash", "-c", "cat public/build/assets-manifest.json | grep entrypoints"])
|
||||
exec=exec_action(["bash", "-c", "stat public/build/assets-manifest.json"]),
|
||||
),
|
||||
allow_parallel=True,
|
||||
labels=["frontend"]
|
||||
labels=["local"]
|
||||
)
|
||||
|
||||
build_backend_env = {}
|
||||
if use_zig:
|
||||
build_backend_env['USE_ZIG'] = "true"
|
||||
|
||||
local_resource(
|
||||
'backend-build',
|
||||
"bash ./build-grafana.sh",
|
||||
deps=[
|
||||
'../../pkg',
|
||||
'../../apps',
|
||||
'../../kinds',
|
||||
'../../kindsv2',
|
||||
'../../local',
|
||||
'../../scripts',
|
||||
'../../conf',
|
||||
'../../go.sum',
|
||||
'../../go.mod',
|
||||
],
|
||||
env=build_backend_env,
|
||||
allow_parallel=True,
|
||||
labels=["local"]
|
||||
)
|
||||
|
||||
# --- Docker Compose
|
||||
docker_compose("./docker-compose.yaml")
|
||||
dc_resource("proxy",
|
||||
resource_deps=["backend", "frontend-service"],
|
||||
labels=["services"]
|
||||
)
|
||||
dc_resource("backend",
|
||||
resource_deps=["yarn start", "backend-build"],
|
||||
labels=["services"]
|
||||
)
|
||||
dc_resource("frontend-service",
|
||||
resource_deps=["yarn start", "backend-build"],
|
||||
labels=["services"],
|
||||
)
|
||||
|
||||
# First argument is the name of the service from the docker-compose file.
|
||||
dc_resource("backend", resource_deps=["yarn start"], labels=["backend"])
|
||||
dc_resource("frontend-service", resource_deps=["yarn start"], labels=["backend"])
|
||||
dc_resource("proxy", resource_deps=["backend", "frontend-service"], labels=["ingress"])
|
||||
# paths in tilt files are confusing....
|
||||
# - if tilt is dealing the the path, it is relative to the Tiltfile
|
||||
# - if docker is dealing with the path, it is relative to the context
|
||||
docker_build('grafana-fs-dev',
|
||||
# Set the docker context to the root of the repo
|
||||
context='../..',
|
||||
dockerfile='grafana-fs-dev.dockerfile',
|
||||
|
||||
docker_build('grafana-backend', '../..',
|
||||
dockerfile='backend.dockerfile',
|
||||
|
||||
# Only these paths will be in the docker context, and will trigger a rebuild.
|
||||
# This must include all the files that are COPY'd in backend.dockerfile
|
||||
# Paths relative to the docker context (root of the repo)
|
||||
only=[
|
||||
"./Makefile",
|
||||
"./devenv/frontend-service/build-grafana.sh",
|
||||
|
||||
"./apps",
|
||||
"./pkg",
|
||||
"./scripts",
|
||||
|
||||
"./go.sum",
|
||||
"./go.mod",
|
||||
"./go.work",
|
||||
"./go.work.sum",
|
||||
|
||||
"./kinds",
|
||||
"./kindsv2",
|
||||
"./public/api-merged.json",
|
||||
"./package.json",
|
||||
|
||||
"./conf/defaults.ini",
|
||||
"./conf/ldap.toml",
|
||||
"./conf/ldap_multiple.toml",
|
||||
"./public/emails",
|
||||
"./public/views",
|
||||
"./public/dashboards",
|
||||
"./public/build/assets-manifest.json",
|
||||
'devenv/frontend-service/build/grafana',
|
||||
'conf/defaults.ini',
|
||||
'public/emails',
|
||||
'public/views',
|
||||
'public/dashboards',
|
||||
'public/app/plugins',
|
||||
'public/build/assets-manifest.json',
|
||||
],
|
||||
|
||||
# Sync paths are relative to the Tiltfile
|
||||
live_update = [
|
||||
sync('./build/grafana', '/grafana/bin/grafana'),
|
||||
sync('../../conf/defaults.ini', '/grafana/conf/defaults.ini'),
|
||||
sync('../../public/emails', '/grafana/public/emails'),
|
||||
sync('../../public/views', '/grafana/public/views'),
|
||||
sync('../../public/dashboards', '/grafana/public/dashboards'),
|
||||
sync('../../public/app/plugins', '/grafana/public/app/plugins'),
|
||||
sync('../../public/build/assets-manifest.json', '/grafana/public/build/assets-manifest.json'),
|
||||
restart_container()
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
docker_build('grafana-proxy', '.',
|
||||
docker_build('grafana-proxy',
|
||||
# Set the docker context to this frontend-service folder
|
||||
context='.',
|
||||
dockerfile='proxy.dockerfile',
|
||||
|
||||
# Path relative to the docker context (this folder)
|
||||
only=[
|
||||
"./nginx.conf",
|
||||
],
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
ARG BASE_IMAGE=alpine:3.21
|
||||
ARG GO_IMAGE=golang:1.24.5-alpine
|
||||
|
||||
# ----- Go build stage
|
||||
FROM ${GO_IMAGE} AS go-dev-builder
|
||||
|
||||
RUN apk add --no-cache \
|
||||
binutils-gold \
|
||||
bash \
|
||||
gcc g++ make git jq findutils
|
||||
|
||||
WORKDIR /build-grafana
|
||||
|
||||
RUN go env GOCACHE
|
||||
RUN go env GOPATH
|
||||
|
||||
# All files COPY'd here must be included in the `only` list in Tiltfile
|
||||
# otherwise the image will not build with Tilt.
|
||||
|
||||
COPY Makefile devenv/frontend-service/build-grafana.sh ./
|
||||
|
||||
# Copy go mod files first
|
||||
# run this command and replace the output below:
|
||||
# find pkg scripts apps -type f \( -name go.mod -o -name go.sum \) -print | sed -E 's#(.*)/go\.(mod|sum)$#COPY \1/go.* \1/#' | sort -u
|
||||
COPY apps/advisor/go.* apps/advisor/
|
||||
COPY apps/alerting/notifications/go.* apps/alerting/notifications/
|
||||
COPY apps/dashboard/go.* apps/dashboard/
|
||||
COPY apps/folder/go.* apps/folder/
|
||||
COPY apps/iam/go.* apps/iam/
|
||||
COPY apps/investigations/go.* apps/investigations/
|
||||
COPY apps/playlist/go.* apps/playlist/
|
||||
COPY apps/secret/go.* apps/secret/
|
||||
COPY pkg/aggregator/go.* pkg/aggregator/
|
||||
COPY pkg/apimachinery/go.* pkg/apimachinery/
|
||||
COPY pkg/apiserver/go.* pkg/apiserver/
|
||||
COPY pkg/build/go.* pkg/build/
|
||||
COPY pkg/build/wire/go.* pkg/build/wire/
|
||||
COPY pkg/codegen/go.* pkg/codegen/
|
||||
COPY pkg/plugins/codegen/go.* pkg/plugins/codegen/
|
||||
COPY pkg/promlib/go.* pkg/promlib/
|
||||
COPY pkg/semconv/go.* pkg/semconv/
|
||||
COPY scripts/go-workspace/go.* scripts/go-workspace/
|
||||
COPY scripts/modowners/go.* scripts/modowners/
|
||||
|
||||
COPY go.* ./
|
||||
|
||||
# Install dependencies
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
go mod download
|
||||
|
||||
# Copy source files
|
||||
COPY kinds kinds
|
||||
COPY kindsv2 kindsv2
|
||||
COPY public/api-merged.json public/api-merged.json
|
||||
COPY apps apps
|
||||
COPY pkg pkg
|
||||
COPY package.json package.json
|
||||
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
--mount=type=cache,target=/root/.cache/go-build \
|
||||
bash build-grafana.sh
|
||||
|
||||
|
||||
# ----- Runtime stage
|
||||
FROM ${BASE_IMAGE}
|
||||
RUN apk add --no-cache ca-certificates tzdata musl-utils bash
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
WORKDIR /grafana
|
||||
|
||||
RUN mkdir -p "conf/provisioning/datasources" \
|
||||
"conf/provisioning/dashboards" \
|
||||
"conf/provisioning/notifiers" \
|
||||
"conf/provisioning/plugins" \
|
||||
"conf/provisioning/access-control" \
|
||||
"conf/provisioning/alerting"
|
||||
|
||||
# Copy config files
|
||||
COPY conf/defaults.ini conf/ldap.toml conf/ldap_multiple.toml conf/
|
||||
|
||||
COPY public/emails public/emails
|
||||
COPY public/views public/views
|
||||
COPY public/dashboards public/dashboards
|
||||
|
||||
# Copy the Go binary from the go-dev-builder stage
|
||||
COPY --from=go-dev-builder /build-grafana/bin/grafana /grafana/bin/grafana
|
||||
|
||||
COPY public/build/assets-manifest.json public/build/assets-manifest.json
|
||||
|
||||
ENTRYPOINT ["bin/grafana", "server"]
|
||||
Regular → Executable
+20
-1
@@ -1,12 +1,31 @@
|
||||
#!/bin/bash
|
||||
|
||||
cd ../../
|
||||
|
||||
echo "Go mod cache: $(go env GOMODCACHE), $(ls -1 $(go env GOMODCACHE) | wc -l) items"
|
||||
echo "Go build cache: $(go env GOCACHE), $(ls -1 $(go env GOCACHE) | wc -l) items"
|
||||
|
||||
# Set cross-compilation env vars only on macOS (Darwin)
|
||||
if [[ "$(uname)" == "Darwin" ]]; then
|
||||
echo "Setting up cross-compilation environment for macOS"
|
||||
export CGO_ENABLED=0
|
||||
export GOOS=linux
|
||||
export GOARCH=arm64
|
||||
fi
|
||||
|
||||
# It's not used by default now that we have CGO-less builds, but keeping this here for a
|
||||
# little bit in case it causes issues for anyone.
|
||||
if [[ -n "$USE_ZIG" ]]; then
|
||||
echo "Using Zig for cross-compilation"
|
||||
export CGO_ENABLED=1
|
||||
export CC="zig cc -target aarch64-linux"
|
||||
export CXX="zig c++ -target aarch64-linux"
|
||||
fi
|
||||
|
||||
# Need to build version into the binary so plugin compatibility works correctly
|
||||
VERSION=$(jq -r .version package.json)
|
||||
|
||||
go build -v \
|
||||
-ldflags "-X main.version=${VERSION}" \
|
||||
-gcflags "all=-N -l" \
|
||||
-o ./bin/grafana ./pkg/cmd/grafana
|
||||
-o ./devenv/frontend-service/build/grafana ./pkg/cmd/grafana
|
||||
@@ -14,24 +14,29 @@ services:
|
||||
- '3010:81' # CDN
|
||||
|
||||
backend:
|
||||
image: grafana-backend
|
||||
image: grafana-fs-dev
|
||||
build:
|
||||
context: .
|
||||
dockerfile: backend.dockerfile
|
||||
context: ../..
|
||||
dockerfile: devenv/frontend-service/grafana-fs-dev.dockerfile
|
||||
entrypoint: ['bin/grafana', 'server']
|
||||
volumes:
|
||||
- backend-data:/grafana/data
|
||||
- ../../public/app/plugins:/grafana/public/app/plugins
|
||||
environment:
|
||||
GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend
|
||||
GF_SERVER_CDN_URL: http://localhost:3010
|
||||
GF_FEATURE_TOGGLES_ENABLE: multiTenantFrontend
|
||||
GF_DATABASE_TYPE: postgres
|
||||
GF_DATABASE_HOST: postgres
|
||||
GF_DATABASE_NAME: grafana
|
||||
GF_DATABASE_USER: grafana
|
||||
GF_DATABASE_PASSWORD: grafana
|
||||
ports:
|
||||
- '3011:3000'
|
||||
|
||||
frontend-service:
|
||||
image: grafana-backend
|
||||
image: grafana-fs-dev
|
||||
build:
|
||||
dockerfile: backend.dockerfile
|
||||
context: ../..
|
||||
dockerfile: devenv/frontend-service/grafana-fs-dev.dockerfile
|
||||
entrypoint: ['bin/grafana', 'server', 'target']
|
||||
ports:
|
||||
- '3012:3000'
|
||||
@@ -41,5 +46,15 @@ services:
|
||||
GF_SECURITY_CONTENT_SECURITY_POLICY: false
|
||||
GF_SERVER_CDN_URL: http://localhost:3010
|
||||
|
||||
postgres:
|
||||
image: postgres:16.1-alpine3.19
|
||||
environment:
|
||||
POSTGRES_USER: grafana
|
||||
POSTGRES_PASSWORD: grafana
|
||||
POSTGRES_DB: grafana
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
|
||||
volumes:
|
||||
backend-data:
|
||||
postgres-data:
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM ubuntu:24.04
|
||||
|
||||
WORKDIR /grafana
|
||||
|
||||
RUN mkdir -p "conf/provisioning/datasources" \
|
||||
"conf/provisioning/dashboards" \
|
||||
"conf/provisioning/notifiers" \
|
||||
"conf/provisioning/plugins" \
|
||||
"conf/provisioning/access-control" \
|
||||
"conf/provisioning/alerting"
|
||||
|
||||
COPY conf/defaults.ini conf/defaults.ini
|
||||
|
||||
COPY public/emails public/emails
|
||||
COPY public/views public/views
|
||||
COPY public/dashboards public/dashboards
|
||||
COPY public/app/plugins public/app/plugins
|
||||
|
||||
ADD devenv/frontend-service/build/grafana bin/grafana
|
||||
|
||||
COPY public/build/assets-manifest.json public/build/assets-manifest.json
|
||||
|
||||
ENTRYPOINT ["bin/grafana", "server"]
|
||||
@@ -19,7 +19,6 @@ if ! tilt version &> /dev/null; then
|
||||
IS_OKAY=false
|
||||
fi
|
||||
|
||||
|
||||
if [ "$IS_OKAY" = false ]; then
|
||||
echo "Please fix the above errors before continuing"
|
||||
exit 1
|
||||
|
||||
Reference in New Issue
Block a user