diff --git a/.drone.yml b/.drone.yml index cc144f74868..9c5f364a85e 100644 --- a/.drone.yml +++ b/.drone.yml @@ -797,6 +797,12 @@ services: volumes: - name: mysql path: /var/lib/mysql +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached steps: - commands: - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST" @@ -892,6 +898,26 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.7.2 name: mysql-integration-tests +- commands: + - dockerize -wait tcp://redis:6379/0 -timeout 120s + - go clean -testcache + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + environment: + REDIS_URL: redis://redis:6379/0 + image: grafana/build-container:1.7.2 + name: redis-integration-tests +- commands: + - dockerize -wait tcp://memcached:11211 -timeout 120s + - go clean -testcache + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + environment: + MEMCACHED_HOSTS: memcached:11211 + image: grafana/build-container:1.7.2 + name: memcached-integration-tests trigger: event: - pull_request @@ -1797,6 +1823,12 @@ services: volumes: - name: mysql path: /var/lib/mysql +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached steps: - commands: - mkdir -p bin @@ -1870,6 +1902,26 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.7.2 name: mysql-integration-tests +- commands: + - dockerize -wait tcp://redis:6379/0 -timeout 120s + - go clean -testcache + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + environment: + REDIS_URL: redis://redis:6379/0 + image: grafana/build-container:1.7.2 + name: redis-integration-tests +- commands: + - dockerize -wait tcp://memcached:11211 -timeout 120s + - go clean -testcache + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + environment: + MEMCACHED_HOSTS: memcached:11211 + image: grafana/build-container:1.7.2 + name: memcached-integration-tests trigger: branch: main event: @@ -4956,6 +5008,12 @@ services: volumes: - name: mysql path: /var/lib/mysql +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached steps: - commands: - mkdir -p bin @@ -5022,6 +5080,26 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.7.2 name: mysql-integration-tests +- commands: + - dockerize -wait tcp://redis:6379/0 -timeout 120s + - go clean -testcache + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + environment: + REDIS_URL: redis://redis:6379/0 + image: grafana/build-container:1.7.2 + name: redis-integration-tests +- commands: + - dockerize -wait tcp://memcached:11211 -timeout 120s + - go clean -testcache + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + environment: + MEMCACHED_HOSTS: memcached:11211 + image: grafana/build-container:1.7.2 + name: memcached-integration-tests trigger: ref: - refs/heads/v[0-9]* @@ -5617,7 +5695,7 @@ services: - name: mysql path: /var/lib/mysql - environment: {} - image: redis:6.2.1-alpine + image: redis:6.2.11-alpine name: redis - environment: {} image: memcached:1.6.9-alpine @@ -5718,8 +5796,7 @@ steps: - commands: - dockerize -wait tcp://redis:6379/0 -timeout 120s - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install environment: @@ -5729,8 +5806,7 @@ steps: - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install environment: @@ -6112,6 +6188,12 @@ services: volumes: - name: mysql path: /var/lib/mysql +- environment: {} + image: redis:6.2.11-alpine + name: redis +- environment: {} + image: memcached:1.6.9-alpine + name: memcached steps: - commands: - mkdir -p bin @@ -6178,6 +6260,26 @@ steps: MYSQL_HOST: mysql image: grafana/build-container:1.7.2 name: mysql-integration-tests +- commands: + - dockerize -wait tcp://redis:6379/0 -timeout 120s + - go clean -testcache + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + environment: + REDIS_URL: redis://redis:6379/0 + image: grafana/build-container:1.7.2 + name: redis-integration-tests +- commands: + - dockerize -wait tcp://memcached:11211 -timeout 120s + - go clean -testcache + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... + depends_on: + - wire-install + environment: + MEMCACHED_HOSTS: memcached:11211 + image: grafana/build-container:1.7.2 + name: memcached-integration-tests trigger: event: - promote @@ -6230,7 +6332,7 @@ services: - name: mysql path: /var/lib/mysql - environment: {} - image: redis:6.2.1-alpine + image: redis:6.2.11-alpine name: redis - environment: {} image: memcached:1.6.9-alpine @@ -6332,8 +6434,7 @@ steps: - commands: - dockerize -wait tcp://redis:6379/0 -timeout 120s - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install environment: @@ -6343,8 +6444,7 @@ steps: - commands: - dockerize -wait tcp://memcached:11211 -timeout 120s - go clean -testcache - - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic - -timeout=5m {}' + - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... depends_on: - wire-install environment: @@ -6653,6 +6753,6 @@ kind: secret name: enterprise2_security_prefix --- kind: signature -hmac: 675c90070f9b68e15f349d079fbafa5288c3e5c30c345a4c9678d69afd1cc5a7 +hmac: 54bded4dc7c3ffbbf3859a4f8f6364f59f3b6e0696152a0b835211eca7119e50 ... diff --git a/Makefile b/Makefile index 5d5d9f4387b..ad161a619d8 100644 --- a/Makefile +++ b/Makefile @@ -143,6 +143,18 @@ test-go-integration-mysql: devenv-mysql ## Run integration tests for mysql backe $(GO) clean -testcache $(GO) list './pkg/...' | xargs -I {} sh -c 'GRAFANA_TEST_DB=mysql go test -run Integration -covermode=atomic -timeout=2m {}' +.PHONY: test-go-integration-redis +test-go-integration-redis: ## Run integration tests for redis cache. + @echo "test backend integration redis tests" + $(GO) clean -testcache + REDIS_URL=localhost:6379 $(GO) test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/... + +.PHONY: test-go-integration-memcached +test-go-integration-memcached: ## Run integration tests for memcached cache. + @echo "test backend integration memcached tests" + $(GO) clean -testcache + MEMCACHED_HOSTS=localhost:11211 $(GO) test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/... + test-js: ## Run tests for frontend. @echo "test frontend" yarn test diff --git a/pkg/infra/remotecache/memcached_storage_integration_test.go b/pkg/infra/remotecache/memcached_storage_integration_test.go index 87fffe182eb..38bf6fc12ea 100644 --- a/pkg/infra/remotecache/memcached_storage_integration_test.go +++ b/pkg/infra/remotecache/memcached_storage_integration_test.go @@ -1,16 +1,23 @@ -//go:build memcached -// +build memcached - package remotecache import ( + "os" "testing" "github.com/grafana/grafana/pkg/setting" ) -func TestMemcachedCacheStorage(t *testing.T) { - opts := &setting.RemoteCacheOptions{Name: memcachedCacheType, ConnStr: "localhost:11211"} +func TestIntegrationMemcachedCacheStorage(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + u, ok := os.LookupEnv("MEMCACHED_HOSTS") + if !ok || u == "" { + t.Skip("No Memcached hosts provided") + } + + opts := &setting.RemoteCacheOptions{Name: memcachedCacheType, ConnStr: u} client := createTestClient(t, opts, nil) runTestsForClient(t, client) runCountTestsForClient(t, opts, nil) diff --git a/pkg/infra/remotecache/redis_storage_integration_test.go b/pkg/infra/remotecache/redis_storage_integration_test.go index d360feb3ce8..392d3bb8b44 100644 --- a/pkg/infra/remotecache/redis_storage_integration_test.go +++ b/pkg/infra/remotecache/redis_storage_integration_test.go @@ -1,17 +1,40 @@ -//go:build redis -// +build redis - package remotecache import ( + "fmt" + "os" + "strings" "testing" + "github.com/go-redis/redis/v8" "github.com/grafana/grafana/pkg/setting" ) -func TestRedisCacheStorage(t *testing.T) { +func TestIntegrationRedisCacheStorage(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } - opts := &setting.RemoteCacheOptions{Name: redisCacheType, ConnStr: "addr=localhost:6379"} + u, ok := os.LookupEnv("REDIS_URL") + if !ok || u == "" { + t.Skip("No redis URL supplied") + } + + addr := u + db := 0 + parsed, err := redis.ParseURL(u) + if err == nil { + addr = parsed.Addr + db = parsed.DB + } + + b := strings.Builder{} + b.WriteString(fmt.Sprintf("addr=%s", addr)) + if db != 0 { + b.WriteString(fmt.Sprintf(",db=%d", db)) + } + + opts := &setting.RemoteCacheOptions{Name: redisCacheType, ConnStr: b.String()} client := createTestClient(t, opts, nil) runTestsForClient(t, client) runCountTestsForClient(t, opts, nil) diff --git a/pkg/infra/remotecache/remotecache_test.go b/pkg/infra/remotecache/remotecache_test.go index 9ad5acbe7a8..9732e8474cd 100644 --- a/pkg/infra/remotecache/remotecache_test.go +++ b/pkg/infra/remotecache/remotecache_test.go @@ -95,7 +95,8 @@ func canPutGetAndDeleteCachedObjects(t *testing.T, client CacheStorage) { assert.Equal(t, err, nil) _, err = client.Get(context.Background(), "key1") - assert.Equal(t, err, ErrCacheItemNotFound) + // redis client returns redis.Nil error when key does not exist. + assert.Error(t, err) } func canNotFetchExpiredItems(t *testing.T, client CacheStorage) { @@ -109,7 +110,8 @@ func canNotFetchExpiredItems(t *testing.T, client CacheStorage) { // should not be able to read that value since its expired _, err = client.Get(context.Background(), "key1") - assert.Equal(t, err, ErrCacheItemNotFound) + // redis client returns redis.Nil error when key does not exist. + assert.Error(t, err) } func TestCollectUsageStats(t *testing.T) { diff --git a/pkg/services/live/managedstream/cache_redis_test.go b/pkg/services/live/managedstream/cache_redis_test.go index df650ded044..d7c3f9fddbb 100644 --- a/pkg/services/live/managedstream/cache_redis_test.go +++ b/pkg/services/live/managedstream/cache_redis_test.go @@ -1,18 +1,34 @@ -//go:build redis -// +build redis - package managedstream import ( + "os" "testing" "github.com/go-redis/redis/v8" "github.com/stretchr/testify/require" ) -func TestRedisCacheStorage(t *testing.T) { +func TestIntegrationRedisCacheStorage(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + u, ok := os.LookupEnv("REDIS_URL") + if !ok || u == "" { + t.Skip("No redis URL supplied") + } + + addr := u + db := 0 + parsed, err := redis.ParseURL(u) + if err == nil { + addr = parsed.Addr + db = parsed.DB + } + redisClient := redis.NewClient(&redis.Options{ - Addr: "localhost:6379", + Addr: addr, + DB: db, }) c := NewRedisFrameCache(redisClient) require.NotNil(t, c) diff --git a/scripts/drone/events/release.star b/scripts/drone/events/release.star index 8c079d33bb7..a163927b045 100644 --- a/scripts/drone/events/release.star +++ b/scripts/drone/events/release.star @@ -141,7 +141,7 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): environment = {"EDITION": "oss"} - services = integration_test_services(edition = "oss") + services = integration_test_services() volumes = integration_test_services_volumes() init_steps = [ @@ -203,6 +203,8 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger): integration_test_steps = [ postgres_integration_tests_step(), mysql_integration_tests_step(), + redis_integration_tests_step(), + memcached_integration_tests_step(), ] # We don't need to run integration tests at release time since they have @@ -282,7 +284,7 @@ def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger): environment = {"EDITION": "enterprise"} - services = integration_test_services(edition = "enterprise") + services = integration_test_services() volumes = integration_test_services_volumes() init_steps = [ @@ -694,11 +696,9 @@ def integration_test_pipelines(): } pipelines = [] volumes = integration_test_services_volumes() - oss_integration_test_steps = [ + integration_test_steps = [ postgres_integration_tests_step(), mysql_integration_tests_step(), - ] - enterprise_integration_test_steps = oss_integration_test_steps + [ redis_integration_tests_step(), memcached_integration_tests_step(), ] @@ -708,7 +708,7 @@ def integration_test_pipelines(): name = "integration-tests-oss", edition = "oss", trigger = trigger, - services = integration_test_services(edition = "oss"), + services = integration_test_services(), steps = [ download_grabpl_step(), identify_runner_step(), @@ -716,7 +716,7 @@ def integration_test_pipelines(): verify_gen_jsonnet_step(), wire_install_step(), ] + - oss_integration_test_steps, + integration_test_steps, environment = {"EDITION": "oss"}, volumes = volumes, )) @@ -725,7 +725,7 @@ def integration_test_pipelines(): name = "integration-tests-enterprise", edition = "enterprise", trigger = trigger, - services = integration_test_services(edition = "enterprise"), + services = integration_test_services(), steps = [ download_grabpl_step(), identify_runner_step(), @@ -744,7 +744,7 @@ def integration_test_pipelines(): [ wire_install_step(), ] + - enterprise_integration_test_steps, + integration_test_steps, environment = {"EDITION": "enterprise"}, volumes = volumes, )) diff --git a/scripts/drone/pipelines/integration_tests.star b/scripts/drone/pipelines/integration_tests.star index 7e8bc47fbd9..87276500809 100644 --- a/scripts/drone/pipelines/integration_tests.star +++ b/scripts/drone/pipelines/integration_tests.star @@ -8,8 +8,10 @@ load( "download_grabpl_step", "enterprise_setup_step", "identify_runner_step", + "memcached_integration_tests_step", "mysql_integration_tests_step", "postgres_integration_tests_step", + "redis_integration_tests_step", "verify_gen_cue_step", "verify_gen_jsonnet_step", "wire_install_step", @@ -37,7 +39,7 @@ def integration_tests(trigger, prefix, ver_mode = "pr"): """ environment = {"EDITION": "oss"} - services = integration_test_services(edition = "oss") + services = integration_test_services() volumes = integration_test_services_volumes() init_steps = [] @@ -66,6 +68,8 @@ def integration_tests(trigger, prefix, ver_mode = "pr"): test_steps = [ postgres_integration_tests_step(), mysql_integration_tests_step(), + redis_integration_tests_step(), + memcached_integration_tests_step(), ] return pipeline( diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index 9d7d04a2ded..d5310032f30 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -8,7 +8,7 @@ def integration_test_services_volumes(): {"name": "mysql", "temp": {"medium": "memory"}}, ] -def integration_test_services(edition): +def integration_test_services(): services = [ { "name": "postgres", @@ -34,24 +34,18 @@ def integration_test_services(edition): }, "volumes": [{"name": "mysql", "path": "/var/lib/mysql"}], }, + { + "name": "redis", + "image": "redis:6.2.11-alpine", + "environment": {}, + }, + { + "name": "memcached", + "image": "memcached:1.6.9-alpine", + "environment": {}, + }, ] - if edition in ("enterprise", "enterprise2"): - services.extend( - [ - { - "name": "redis", - "image": "redis:6.2.1-alpine", - "environment": {}, - }, - { - "name": "memcached", - "image": "memcached:1.6.9-alpine", - "environment": {}, - }, - ], - ) - return services def ldap_service(): diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index 82d61f390fd..749250171a6 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -1144,7 +1144,7 @@ def redis_integration_tests_step(): "commands": [ "dockerize -wait tcp://redis:6379/0 -timeout 120s", "go clean -testcache", - "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", + "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...", ], } @@ -1159,7 +1159,7 @@ def memcached_integration_tests_step(): "commands": [ "dockerize -wait tcp://memcached:11211 -timeout 120s", "go clean -testcache", - "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'", + "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...", ], }