diff --git a/pkg/modules/dependencies.go b/pkg/modules/dependencies.go index 546f0f2fc83..e413f5248c4 100644 --- a/pkg/modules/dependencies.go +++ b/pkg/modules/dependencies.go @@ -16,6 +16,7 @@ const ( ) var dependencyMap = map[string][]string{ + MemberlistKV: {InstrumentationServer}, StorageRing: {InstrumentationServer, MemberlistKV}, GrafanaAPIServer: {InstrumentationServer}, StorageServer: {InstrumentationServer, StorageRing}, diff --git a/pkg/storage/unified/README.md b/pkg/storage/unified/README.md index eeb06364c0c..84efac5df86 100644 --- a/pkg/storage/unified/README.md +++ b/pkg/storage/unified/README.md @@ -274,3 +274,198 @@ unified storage when the dual writer mode is set to 3 or greater. When <= 2, the ## Running load tests Load tests and instructions can be found [here](https://github.com/grafana/grafana-api-tests/tree/main/simulation/src/unified_storage). + +## Running with a distributor + +For this deployment model, the storage-api server establishes a consistent hashing ring to distribute tenant requests. The distributor serves as the primary request router, mapping incoming traffic to the appropriate storage-api server based on tenant ID. When testing functionalities reliant on this sharded persistence layer, the following steps are mandatory. + +### 0. Update your network interface to allow processes to bind to localhost addresses + +For this setup to work, we need to have more than one instance of `storage-api` and at least one instance of +`distributor` service. This step is a requirement for MacOS, as it by default will only allow processes to bind to `127.0.0.1` and not +`127.0.0.2`. + +Run the command below in your terminal for every IP you want to enable: + +```sh +sudo ifconfig lo0 alias up +``` + +### 1. Start MySQL DB + +The storage server doesn't support `sqlite` so we need to have a dedicated external database. You can start one with +docker in case you don't have one: + +```sh +docker run -d --name db -e "MYSQL_DATABASE=grafana" -e "MYSQL_USER=grafana" -e "MYSQL_PASSWORD=grafana" -e "MYSQL_ROOT_PASSWORD=root" -p 3306:3306 docker.io/bitnami/mysql:8.0.31 +``` + +### 2. Create dedicated ini files for every service + +Example distributor ini file: + +* Bind grpc/http server to `127.0.0.1` +* Bind and join `memberlist` on `127.0.0.1:7946` (default memberlist port) + +```ini +target = distributor + +[server] +http_port = 3000 +http_addr = "127.0.0.1" + +[grpc_server] +network = "tcp" +address = "127.0.0.1:10000" + +[grafana-apiserver] +storage_type = unified + +[grpc_server_authentication] +signing_keys_url = http://localhost:3011/api/signing-keys/keys +mode = "on-prem" + +[unified_storage] +enable_sharding = true +memberlist_bind_addr = "127.0.0.1" +memberlist_advertise_addr = "127.0.0.1" +memberlist_join_member = "127.0.0.1:7946" +``` + +Example unified storage ini file: + +* Bind grpc/http server to `127.0.0.2` +* Configue MySQL database parameters +* Enable a few feature flags +* Give it a unique `instance_id` (defaults to hostname, so you need to define it locally) +* Bind `memberlist` to `127.0.0.2` and join the member on `127.0.0.1` (the distributor module above) + +You can repeat the same configuration for many different storage-api instances by changing the bind address +from `127.0.0.2` to something else, eg `127.0.0.3` + +```ini +target = storage-server + +[server] +http_port = 3000 +http_addr = "127.0.0.2" + +[resource_api] +db_type = mysql +db_host = localhost:3306 +db_name = grafana ; or whatever you defined in your currently running database +db_user = grafana ; or whatever you defined in your currently running database +db_pass = grafana ; or whatever you defined in your currently running database + +[grpc_server] +network = "tcp" +address = "127.0.0.2:10000" + +[grafana-apiserver] +storage_type = unified + +[grpc_server_authentication] +signing_keys_url = http://localhost:3011/api/signing-keys/keys +mode = "on-prem" + +[feature_toggles] +kubernetesClientDashboardsFolders = true +kubernetesDashboardsAPI = true +kubernetesFolders = true +unifiedStorage = true +unifiedStorageSearch = true + +[unified_storage] +enable_sharding = true +instance_id = node-0 +memberlist_bind_addr = "127.0.0.2" +memberlist_advertise_addr = "127.0.0.2" +memberlist_join_member = "127.0.0.1:7946" +``` + +Example grafana ini file: + +* Bind http server to `127.0.0.2`. +* Explicitly declare the sqlite db. This is so when you run a second instance they don't both try to use the same sqlite + file. +* Configure the storage api client to talk to the distributor on `127.0.0.1:10000` +* Configure feature flags/modes as desired + +Then repeat this configuration and change: + +* the `stack_id` to something unique +* the database +* the bind address (so the browser can save the auth for every instance in a different cookie) +```ini +target = all + +[environment] +stack_id = 1 + +[database] +type = sqlite3 +name = grafana +user = root +path = grafana1.db + +[grafana-apiserver] +address = 127.0.0.1:10000 +storage_type = unified-grpc + +[server] +protocol = http +http_port = 3011 +http_addr = "127.0.0.2" + +[feature_toggles] +kubernetesClientDashboardsFolders = true +kubernetesDashboardsAPI = true +kubernetesFolders = true +unifiedStorageSearchUI = true + +[unified_storage.dashboards.dashboard.grafana.app] +dualWriterMode = 3 +[unified_storage.folders.folder.grafana.app] +dualWriterMode = 3 +[unified_storage.playlists.playlist.grafana.app] +dualWriterMode = 4 +``` + +### 3. Run the services + +Build the backend: + +```sh +GO_BUILD_DEV=1 make build-go +``` + +You will need a separate process for every service. It's the same command with a separate `ini` file to it. For +example, if you created a `distributor.ini` file in the `conf` directory, this is how you would run the distributor: + +```sh +./bin/grafana server target --config conf/distributor.ini +``` + +Repeat for the other services. + +```sh +./bin/grafana server target --config conf/storage-api-1.ini +./bin/grafana server target --config conf/storage-api-2.ini +./bin/grafana server target --config conf/storage-api-3.ini + +./bin/grafana server target --config conf/grafana1.ini +./bin/grafana server target --config conf/grafana2.ini +./bin/grafana server target --config conf/grafana3.ini +``` + +etc + +### 4. Verify that it is working + +If all is well, you will be able to visit every grafana stack you started and use it normally. Visit +`http://127.0.0.2:3011`, login with `admin`/`admin`, create some dashboards/folders, etc. + +For debugging purposes, you can view the memberlist status by visitting `http://127.0.0.1:3000/memberlist` and check +that every instance you create is part of the memberlist. +You can also visit `http://127.0.0.1:3000/ring` to view the ring status and the storage-api servers that are part of the +ring.