test(unified-storage): Create initial e2e test scenario for unified storage (#102019)
* test: introduce initial e2e test scenario for unified storage * fix: lint * chore: trigger build * test: attach unified storage grpc service into grafana * fix: lint and todos * fix: err handling * chore: trigger build
This commit is contained in:
Mustafa Sencer Özcan
@@ -0,0 +1,87 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
|
||||
"github.com/grafana/e2e"
|
||||
gapi "github.com/grafana/grafana-api-golang-client"
|
||||
)
|
||||
|
||||
const (
|
||||
grafanaINI = `
|
||||
app_mode = development
|
||||
target = all
|
||||
|
||||
[grafana-apiserver]
|
||||
address = 0.0.0.0:10000
|
||||
storage_type = unified-grpc
|
||||
|
||||
[unified_storage.dashboards.dashboard.grafana.app]
|
||||
dualWriterMode = 5
|
||||
|
||||
[unified_storage.folders.folder.grafana.app]
|
||||
dualWriterMode = 5
|
||||
`
|
||||
defaultGrafanaImage = "grafana/grafana:main"
|
||||
grafanaBinary = "/run.sh"
|
||||
grafanaHTTPPort = 3000
|
||||
grafanaGRPCPort = 10000
|
||||
)
|
||||
|
||||
type GrafanaService struct {
|
||||
*e2e.HTTPService
|
||||
}
|
||||
|
||||
func NewGrafanaService(name string, flags, envVars map[string]string) *GrafanaService {
|
||||
svc := &GrafanaService{
|
||||
HTTPService: e2e.NewHTTPService(
|
||||
name,
|
||||
GetGrafanaImage(),
|
||||
e2e.NewCommandWithoutEntrypoint(grafanaBinary, e2e.BuildArgs(flags)...),
|
||||
e2e.NewHTTPReadinessProbe(grafanaHTTPPort, "/ready", 200, 299),
|
||||
grafanaHTTPPort,
|
||||
grafanaGRPCPort),
|
||||
}
|
||||
|
||||
svc.ConcreteService.SetEnvVars(envVars)
|
||||
|
||||
return svc
|
||||
}
|
||||
|
||||
func (g *GrafanaService) GRPCEndpoint() string {
|
||||
return g.HTTPService.NetworkEndpoint(grafanaGRPCPort)
|
||||
}
|
||||
|
||||
type GrafanaClient struct {
|
||||
*gapi.Client
|
||||
}
|
||||
|
||||
func NewGrafanaClient(host string, orgID int64) (*GrafanaClient, error) {
|
||||
cfg := gapi.Config{
|
||||
BasicAuth: url.UserPassword("admin", "admin"),
|
||||
OrgID: orgID,
|
||||
HTTPHeaders: map[string]string{
|
||||
"X-Disable-Provenance": "true",
|
||||
},
|
||||
}
|
||||
|
||||
client, err := gapi.New(fmt.Sprintf("http://%s/", host), cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &GrafanaClient{
|
||||
Client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetDefaultImage returns the Docker image to use to run the Grafana..
|
||||
func GetGrafanaImage() string {
|
||||
if img := os.Getenv("GRAFANA_IMAGE"); img != "" {
|
||||
return img
|
||||
}
|
||||
|
||||
return defaultGrafanaImage
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/grafana/e2e"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPostgresImage = "postgres:16.4"
|
||||
postgresHTTPPort = 5432
|
||||
)
|
||||
|
||||
// GetDefaultImage returns the Docker image to use to run the Postgres..
|
||||
func GetPostgresImage() string {
|
||||
if img := os.Getenv("POSTGRES_IMAGE"); img != "" {
|
||||
return img
|
||||
}
|
||||
|
||||
return defaultPostgresImage
|
||||
}
|
||||
|
||||
type PostgresService struct {
|
||||
*e2e.HTTPService
|
||||
}
|
||||
|
||||
func NewPostgresService(name string, envVars map[string]string) *PostgresService {
|
||||
svc := &PostgresService{
|
||||
HTTPService: e2e.NewHTTPService(
|
||||
name,
|
||||
GetPostgresImage(),
|
||||
nil,
|
||||
nil,
|
||||
postgresHTTPPort,
|
||||
),
|
||||
}
|
||||
|
||||
svc.SetEnvVars(envVars)
|
||||
|
||||
return svc
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"github.com/grafana/e2e"
|
||||
)
|
||||
|
||||
const (
|
||||
storageINI = `
|
||||
app_mode = development
|
||||
target = storage-server
|
||||
|
||||
[grafana-apiserver]
|
||||
storage_type = unified
|
||||
|
||||
[database]
|
||||
type = postgres
|
||||
host = postgres:5432
|
||||
name = grafana
|
||||
user = admin
|
||||
password = admin
|
||||
ssl_mode = disable
|
||||
`
|
||||
)
|
||||
|
||||
type StorageService struct {
|
||||
*e2e.ConcreteService
|
||||
}
|
||||
|
||||
func NewStorageService(name string, flags, envVars map[string]string) *StorageService {
|
||||
args := append([]string{"target"}, e2e.BuildArgs(flags)...)
|
||||
svc := &StorageService{
|
||||
ConcreteService: e2e.NewConcreteService(
|
||||
name,
|
||||
GetGrafanaImage(),
|
||||
e2e.NewCommandWithoutEntrypoint(grafanaBinary, args...),
|
||||
e2e.NewTCPReadinessProbe(10000),
|
||||
10000,
|
||||
),
|
||||
}
|
||||
|
||||
svc.SetEnvVars(envVars)
|
||||
svc.Endpoint(10000)
|
||||
|
||||
return svc
|
||||
}
|
||||
|
||||
// TODO: remove this method when the storage-server module has a graceful shutdown on docker stop
|
||||
// the following method is implemented because the storage-server module is not shutting down gracefully
|
||||
// https://github.com/grafana/search-and-storage-team/issues/228
|
||||
func (s *StorageService) Stop() error {
|
||||
_ = s.ConcreteService.Stop()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *StorageService) GRPCEndpoint() string {
|
||||
return s.NetworkEndpoint(grafanaGRPCPort)
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/grafana/e2e"
|
||||
)
|
||||
|
||||
type UnifiedScenario struct {
|
||||
*e2e.Scenario
|
||||
|
||||
Grafana *GrafanaService
|
||||
UnifiedStorage *StorageService
|
||||
Postgres *PostgresService
|
||||
}
|
||||
|
||||
func NewUnifiedScenario() (*UnifiedScenario, error) {
|
||||
s, err := e2e.NewScenario(GetNetworkName())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &UnifiedScenario{
|
||||
Scenario: s,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *UnifiedScenario) NewGrafanaService(name string, grpcEndpoint string) (*GrafanaService, error) {
|
||||
flags := map[string]string{
|
||||
"--config": fmt.Sprintf("/shared/%s.ini", name),
|
||||
}
|
||||
fts := []string{
|
||||
"grafanaAPIServerWithExperimentalAPIs",
|
||||
"kubernetesClientDashboardsFolders",
|
||||
"grafanaAPIServerEnsureKubectlAccess",
|
||||
"unifiedStorageSearch",
|
||||
"unifiedStorageSearchUI",
|
||||
"kubernetesCliDashboards",
|
||||
"kubernetesDashboardsAPI",
|
||||
"kubernetesFolders",
|
||||
"unified_storage",
|
||||
}
|
||||
envVars := map[string]string{
|
||||
"GF_PATHS_CONFIG": fmt.Sprintf("/shared/%s.ini", name),
|
||||
"GF_FEATURE_TOGGLES_ENABLE": strings.Join(fts, ","),
|
||||
"GF_GRAFANA_APISERVER_ADDRESS": grpcEndpoint,
|
||||
"GF_GRAFANA_APISERVER_STORAGE_TYPE": "unified-grpc",
|
||||
"GF_DATABASE_TYPE": "postgres",
|
||||
"GF_DATABASE_HOST": "postgres:5432",
|
||||
"GF_DATABASE_NAME": "grafana",
|
||||
"GF_DATABASE_USER": "admin",
|
||||
"GF_DATABASE_PASSWORD": "admin",
|
||||
"GF_DATABASE_SSL_MODE": "disable",
|
||||
}
|
||||
|
||||
if err := s.loadCfg(name, grafanaINI); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g := NewGrafanaService(name, flags, envVars)
|
||||
s.Grafana = g
|
||||
return g, nil
|
||||
}
|
||||
|
||||
func (s *UnifiedScenario) NewGrafanaClient(grafanaName string, orgID int64) (*GrafanaClient, error) {
|
||||
g := s.Grafana
|
||||
return NewGrafanaClient(g.HTTPEndpoint(), orgID)
|
||||
}
|
||||
|
||||
func (s *UnifiedScenario) NewStorageService(name string) (*StorageService, error) {
|
||||
flags := map[string]string{
|
||||
"--config": "/shared/storage.ini",
|
||||
}
|
||||
envVars := map[string]string{
|
||||
"GF_PATHS_CONFIG": "/shared/storage.ini",
|
||||
"GF_DEFAULT_TARGET": "storage-server",
|
||||
"GF_GRPC_SERVER_ADDRESS": "0.0.0.0:10000",
|
||||
"GF_FEATURE_TOGGLES_ENABLE": "unifiedStorage,unifiedStorageSearch",
|
||||
"GF_RESOURCE_API_DB_TYPE": "postgres",
|
||||
"GF_RESOURCE_API_DB_HOST": "postgres:5432",
|
||||
"GF_RESOURCE_API_DB_NAME": "grafana",
|
||||
"GF_RESOURCE_API_DB_USER": "admin",
|
||||
"GF_RESOURCE_API_DB_PASS": "admin",
|
||||
"GF_DATABASE_TYPE": "postgres",
|
||||
"GF_DATABASE_HOST": "postgres:5432",
|
||||
"GF_DATABASE_NAME": "grafana",
|
||||
"GF_DATABASE_USER": "admin",
|
||||
"GF_DATABASE_PASSWORD": "admin",
|
||||
"GF_DATABASE_SSL_MODE": "disable",
|
||||
}
|
||||
|
||||
if err := s.loadCfg(name, storageINI); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
us := NewStorageService(name, flags, envVars)
|
||||
s.UnifiedStorage = us
|
||||
return us, nil
|
||||
}
|
||||
|
||||
func (s *UnifiedScenario) NewPostgresService(name string) *PostgresService {
|
||||
ps := NewPostgresService(name, map[string]string{
|
||||
"POSTGRES_USER": "admin",
|
||||
"POSTGRES_PASSWORD": "admin",
|
||||
"POSTGRES_DB": "grafana",
|
||||
})
|
||||
s.Postgres = ps
|
||||
|
||||
return ps
|
||||
}
|
||||
|
||||
func (s *UnifiedScenario) loadCfg(svc, content string) error {
|
||||
dst := fmt.Sprintf("%s/%s.ini", s.SharedDir(), svc)
|
||||
destination, err := os.Create(filepath.Clean(dst))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = destination.Write([]byte(content))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
defaultNetworkName = "e2e-grafana-unified"
|
||||
)
|
||||
|
||||
// GetNetworkName returns the docker network name to run tests within.
|
||||
func GetNetworkName() string {
|
||||
// If the E2E_NETWORK_NAME is set, use that for the network name.
|
||||
// Otherwise, return the default network name.
|
||||
if os.Getenv("E2E_NETWORK_NAME") != "" {
|
||||
return os.Getenv("E2E_NETWORK_NAME")
|
||||
}
|
||||
|
||||
return defaultNetworkName
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
gapi "github.com/grafana/grafana-api-golang-client"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestUnifiedStorage(t *testing.T) {
|
||||
// TODO: remove this line when ci pipeline is ready
|
||||
t.Skip("skipping e2e test")
|
||||
|
||||
t.Run("Create folder and dashboard", func(t *testing.T) {
|
||||
s, err := NewUnifiedScenario()
|
||||
require.NoError(t, err)
|
||||
|
||||
postgres := s.NewPostgresService("postgres")
|
||||
require.NoError(t, s.Scenario.StartAndWaitReady(postgres))
|
||||
|
||||
storage, err := s.NewStorageService("storage")
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.Scenario.StartAndWaitReady(storage))
|
||||
|
||||
grafana, err := s.NewGrafanaService("grafana", storage.GRPCEndpoint())
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, s.Scenario.StartAndWaitReady(grafana))
|
||||
|
||||
const org1 = 1
|
||||
gorg1, err := s.NewGrafanaClient("grafana", org1)
|
||||
require.NoError(t, err)
|
||||
|
||||
folderOrg1, err := gorg1.NewFolder("test-folder-1")
|
||||
require.NoError(t, err)
|
||||
|
||||
const uid = "test-dashboard-1"
|
||||
dashboard := gapi.Dashboard{
|
||||
Model: map[string]interface{}{
|
||||
"uid": uid,
|
||||
"title": "Test Dashboard 1",
|
||||
},
|
||||
FolderID: folderOrg1.ID,
|
||||
FolderUID: folderOrg1.UID,
|
||||
Overwrite: false,
|
||||
Message: "Test Dashboard 1",
|
||||
}
|
||||
|
||||
dashboardOrg1, err := gorg1.NewDashboard(dashboard)
|
||||
require.NoError(t, err)
|
||||
|
||||
newDashboard, err := gorg1.DashboardByUID(uid)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, dashboardOrg1.UID, newDashboard.Model["uid"])
|
||||
|
||||
newFolder, err := gorg1.Folder(folderOrg1.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, folderOrg1.UID, newFolder.UID)
|
||||
|
||||
require.NoError(t, s.Scenario.Stop(grafana, storage, postgres))
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user