public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Storage: Add basic file upload management (#50638)

Browse Source
This commit is contained in:
Ryan McKinley
2022-07-05 10:53:41 -07:00
committed by GitHub
parent 4a76436be2
commit 4a00c7ebde
25 changed files with 1105 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/api/api.go
+1
View File
@@ -75,6 +75,7 @@ func (hs *HTTPServer) registerRoutes() {
r.Get("/admin/orgs", authorizeInOrg(reqGrafanaAdmin, ac.UseGlobalOrg, orgsAccessEvaluator), hs.Index)
r.Get("/admin/orgs/edit/:id", authorizeInOrg(reqGrafanaAdmin, ac.UseGlobalOrg, orgsAccessEvaluator), hs.Index)
r.Get("/admin/stats", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionServerStatsRead)), hs.Index)
r.Get("/admin/storage/*", reqGrafanaAdmin, hs.Index)
r.Get("/admin/ldap", authorize(reqGrafanaAdmin, ac.EvalPermission(ac.ActionLDAPStatusRead)), hs.Index)
r.Get("/styleguide", reqSignedIn, hs.Index)
pkg/api/index.go
+11
View File
@@ -294,6 +294,7 @@ func (hs *HTTPServer) getNavTree(c *models.ReqContext, hasEditPerm bool, prefs *
Url: hs.Cfg.AppSubURL + "/org/apikeys",
})
}
if enableServiceAccount(hs, c) {
configNodes = append(configNodes, &dtos.NavLink{
Text: "Service accounts",
@@ -646,6 +647,16 @@ func (hs *HTTPServer) buildAdminNavLinks(c *models.ReqContext) []*dtos.NavLink {
})
}
if hasAccess(ac.ReqGrafanaAdmin, ac.EvalPermission(ac.ActionSettingsRead)) && hs.Features.IsEnabled(featuremgmt.FlagStorage) {
adminNavLinks = append(adminNavLinks, &dtos.NavLink{
Text: "Storage",
Id: "storage",
Description: "Manage file storage",
Icon: "cube",
Url: hs.Cfg.AppSubURL + "/admin/storage",
})
}
if hs.Cfg.LDAPEnabled && hasAccess(ac.ReqGrafanaAdmin, ac.EvalPermission(ac.ActionLDAPStatusRead)) {
adminNavLinks = append(adminNavLinks, &dtos.NavLink{
Text: "LDAP", Id: "ldap", Url: hs.Cfg.AppSubURL + "/admin/ldap", Icon: "book",
pkg/services/store/config.go
+4 -3
View File
@@ -1,9 +1,10 @@
package store
type RootStorageConfig struct {
Type string `json:"type"`
Prefix string `json:"prefix"`
Name string `json:"name"`
Type string `json:"type"`
Prefix string `json:"prefix"`
Name string `json:"name"`
Description string `json:"description"`
// Depending on type, these will be configured
Disk *StorageLocalDiskConfig `json:"disk,omitempty"`
pkg/services/store/http.go
+5 -2
View File
@@ -2,12 +2,14 @@ package store
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"strings"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/util"
"github.com/grafana/grafana/pkg/web"
)
@@ -57,7 +59,8 @@ func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
}
c.Req.Body = http.MaxBytesReader(c.Resp, c.Req.Body, MAX_UPLOAD_SIZE)
if err := c.Req.ParseMultipartForm(MAX_UPLOAD_SIZE); err != nil {
return response.Error(400, "Please limit file uploaded under 1MB", err)
msg := fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE))
return response.Error(400, msg, err)
}
files := c.Req.MultipartForm.File["file"]
@@ -92,7 +95,7 @@ func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
return errFileTooBig
}
path := RootUpload + "/" + fileHeader.Filename
path := RootResources + "/" + fileHeader.Filename
mimeType := http.DetectContentType(data)
pkg/services/store/service.go
+15 -11
View File
@@ -26,9 +26,9 @@ var ErrValidationFailed = errors.New("request validation failed")
var ErrFileAlreadyExists = errors.New("file exists")
const RootPublicStatic = "public-static"
const RootUpload = "upload"
const RootResources = "resources"
const MAX_UPLOAD_SIZE = 1024 * 1024 // 1MB
const MAX_UPLOAD_SIZE = 3 * 1024 * 1024 // 3MB
type StorageService interface {
registry.BackgroundService
@@ -60,21 +60,25 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
Path: cfg.StaticRootPath,
Roots: []string{
"/testdata/",
// "/img/icons/",
// "/img/bg/",
"/img/",
"/gazetteer/",
"/maps/",
},
}).setReadOnly(true).setBuiltin(true),
}).setReadOnly(true).setBuiltin(true).
setDescription("Access files from the static public files"),
}
initializeOrgStorages := func(orgId int64) []storageRuntime {
storages := make([]storageRuntime, 0)
if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) {
config := &StorageSQLConfig{orgId: orgId}
storages = append(storages, newSQLStorage(RootUpload, "Local file upload", config, sql).setBuiltin(true))
storages = append(storages,
newSQLStorage(RootResources,
"Resources",
&StorageSQLConfig{orgId: orgId}, sql).
setBuiltin(true).
setDescription("Upload custom resource files"))
}
return storages
}
@@ -133,16 +137,16 @@ type UploadRequest struct {
}
func (s *standardStorageService) Upload(ctx context.Context, user *models.SignedInUser, req *UploadRequest) error {
upload, _ := s.tree.getRoot(getOrgId(user), RootUpload)
upload, _ := s.tree.getRoot(getOrgId(user), RootResources)
if upload == nil {
return ErrUploadFeatureDisabled
}
if !strings.HasPrefix(req.Path, RootUpload+"/") {
if !strings.HasPrefix(req.Path, RootResources+"/") {
return ErrUnsupportedStorage
}
storagePath := strings.TrimPrefix(req.Path, RootUpload)
storagePath := strings.TrimPrefix(req.Path, RootResources)
validationResult := s.validateUploadRequest(ctx, user, req, storagePath)
if !validationResult.ok {
grafanaStorageLogger.Warn("file upload validation failed", "filetype", req.MimeType, "path", req.Path, "reason", validationResult.reason)
@@ -178,7 +182,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
}
func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error {
upload, _ := s.tree.getRoot(getOrgId(user), RootUpload)
upload, _ := s.tree.getRoot(getOrgId(user), RootResources)
if upload == nil {
return fmt.Errorf("upload feature is not enabled")
}
pkg/services/store/service_test.go
+1 -1
View File
@@ -63,7 +63,7 @@ func TestUpload(t *testing.T) {
request := UploadRequest{
EntityType: EntityTypeImage,
Contents: make([]byte, 0),
Path: "upload/myFile.jpg",
Path: "resources/myFile.jpg",
MimeType: "image/jpg",
}
err = s.Upload(context.Background(), dummyUser, &request)
pkg/services/store/tree.go
+36 -11
View File
@@ -89,31 +89,52 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (
if path == "" || path == "/" {
t.assureOrgIsInitialized(orgId)
idx := 0
count := len(t.rootsByOrgId[ac.GlobalOrgID])
if orgId != ac.GlobalOrgID {
count += len(t.rootsByOrgId[orgId])
}
title := data.NewFieldFromFieldType(data.FieldTypeString, count)
names := data.NewFieldFromFieldType(data.FieldTypeString, count)
title := data.NewFieldFromFieldType(data.FieldTypeString, count)
descr := data.NewFieldFromFieldType(data.FieldTypeString, count)
types := data.NewFieldFromFieldType(data.FieldTypeString, count)
readOnly := data.NewFieldFromFieldType(data.FieldTypeBool, count)
builtIn := data.NewFieldFromFieldType(data.FieldTypeBool, count)
mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
title.Name = "title"
names.Name = "name"
descr.Name = "description"
mtype.Name = "mediaType"
for i, f := range t.rootsByOrgId[ac.GlobalOrgID] {
names.Set(i, f.Meta().Config.Prefix)
title.Set(i, f.Meta().Config.Name)
mtype.Set(i, "directory")
types.Name = "storageType"
readOnly.Name = "readOnly"
builtIn.Name = "builtIn"
for _, f := range t.rootsByOrgId[ac.GlobalOrgID] {
meta := f.Meta()
names.Set(idx, meta.Config.Prefix)
title.Set(idx, meta.Config.Name)
descr.Set(idx, meta.Config.Description)
mtype.Set(idx, "directory")
types.Set(idx, meta.Config.Type)
readOnly.Set(idx, meta.ReadOnly)
builtIn.Set(idx, meta.Builtin)
idx++
}
if orgId != ac.GlobalOrgID {
for i, f := range t.rootsByOrgId[orgId] {
names.Set(i, f.Meta().Config.Prefix)
title.Set(i, f.Meta().Config.Name)
mtype.Set(i, "directory")
for _, f := range t.rootsByOrgId[orgId] {
meta := f.Meta()
names.Set(idx, meta.Config.Prefix)
title.Set(idx, meta.Config.Name)
descr.Set(idx, meta.Config.Description)
mtype.Set(idx, "directory")
types.Set(idx, meta.Config.Type)
readOnly.Set(idx, meta.ReadOnly)
builtIn.Set(idx, meta.Builtin)
idx++
}
}
frame := data.NewFrame("", names, title, mtype)
frame := data.NewFrame("", names, title, descr, mtype, types, readOnly, builtIn)
frame.SetMeta(&data.FrameMeta{
Type: data.FrameTypeDirectoryListing,
})
@@ -125,7 +146,11 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (
return nil, nil // not found (or not ready)
}
listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{Recursive: false, WithFolders: true, WithFiles: true})
listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{
Recursive: false,
WithFolders: true,
WithFiles: true,
})
if err != nil {
return nil, err
pkg/services/store/types.go
+5
View File
@@ -82,6 +82,11 @@ func (t *baseStorageRuntime) setBuiltin(val bool) *baseStorageRuntime {
return t
}
func (t *baseStorageRuntime) setDescription(v string) *baseStorageRuntime {
t.meta.Config.Description = v
return t
}
type RootStorageMeta struct {
ReadOnly bool `json:"editable,omitempty"`
Builtin bool `json:"builtin,omitempty"`
pkg/util/strings.go
+14
View File
@@ -117,3 +117,17 @@ func Capitalize(s string) string {
r[0] = unicode.ToUpper(r[0])
return string(r)
}
func ByteCountSI(b int64) string {
const unit = 1000
if b < unit {
return fmt.Sprintf("%d B", b)
}
div, exp := int64(unit), 0
for n := b / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB",
float64(b)/float64(div), "kMGTPE"[exp])
}