Storage: add basic storage service (#46604)
This commit is contained in:
Ryan McKinley
@@ -1,20 +1,18 @@
|
||||
package grafanads
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend"
|
||||
"github.com/grafana/grafana-plugin-sdk-go/data"
|
||||
"github.com/grafana/grafana-plugin-sdk-go/experimental"
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/searchV2"
|
||||
"github.com/grafana/grafana/pkg/services/store"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/tsdb/testdatasource"
|
||||
)
|
||||
@@ -35,27 +33,18 @@ const DatasourceUID = "grafana"
|
||||
// This is important to do since otherwise we will only get a
|
||||
// not implemented error response from plugin at runtime.
|
||||
var (
|
||||
_ backend.QueryDataHandler = (*Service)(nil)
|
||||
_ backend.CheckHealthHandler = (*Service)(nil)
|
||||
logger = log.New("tsdb.grafana")
|
||||
_ backend.QueryDataHandler = (*Service)(nil)
|
||||
_ backend.CheckHealthHandler = (*Service)(nil)
|
||||
)
|
||||
|
||||
func ProvideService(cfg *setting.Cfg, search searchV2.SearchService) *Service {
|
||||
return newService(cfg, search)
|
||||
func ProvideService(cfg *setting.Cfg, search searchV2.SearchService, store store.StorageService) *Service {
|
||||
return newService(cfg, search, store)
|
||||
}
|
||||
|
||||
func newService(cfg *setting.Cfg, search searchV2.SearchService) *Service {
|
||||
func newService(cfg *setting.Cfg, search searchV2.SearchService, store store.StorageService) *Service {
|
||||
s := &Service{
|
||||
staticRootPath: cfg.StaticRootPath,
|
||||
roots: []string{
|
||||
"testdata",
|
||||
"img/icons",
|
||||
"img/bg",
|
||||
"gazetteer",
|
||||
"maps",
|
||||
"upload", // does not exist yet
|
||||
},
|
||||
search: search,
|
||||
store: store,
|
||||
}
|
||||
|
||||
return s
|
||||
@@ -63,10 +52,8 @@ func newService(cfg *setting.Cfg, search searchV2.SearchService) *Service {
|
||||
|
||||
// Service exists regardless of user settings
|
||||
type Service struct {
|
||||
// path to the public folder
|
||||
staticRootPath string
|
||||
roots []string
|
||||
search searchV2.SearchService
|
||||
search searchV2.SearchService
|
||||
store store.StorageService
|
||||
}
|
||||
|
||||
func DataSourceModel(orgId int64) *models.DataSource {
|
||||
@@ -89,9 +76,9 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
|
||||
case queryTypeRandomWalk:
|
||||
response.Responses[q.RefID] = s.doRandomWalk(q)
|
||||
case queryTypeList:
|
||||
response.Responses[q.RefID] = s.doListQuery(q)
|
||||
response.Responses[q.RefID] = s.doListQuery(ctx, q)
|
||||
case queryTypeRead:
|
||||
response.Responses[q.RefID] = s.doReadQuery(q)
|
||||
response.Responses[q.RefID] = s.doReadQuery(ctx, q)
|
||||
case queryTypeSearch:
|
||||
response.Responses[q.RefID] = s.doSearchQuery(ctx, req, q)
|
||||
default:
|
||||
@@ -111,25 +98,7 @@ func (s *Service) CheckHealth(_ context.Context, _ *backend.CheckHealthRequest)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) publicPath(path string) (string, error) {
|
||||
if strings.Contains(path, "..") {
|
||||
return "", fmt.Errorf("invalid string")
|
||||
}
|
||||
|
||||
ok := false
|
||||
for _, root := range s.roots {
|
||||
if strings.HasPrefix(path, root) {
|
||||
ok = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return "", fmt.Errorf("bad root path")
|
||||
}
|
||||
return filepath.Join(s.staticRootPath, path), nil
|
||||
}
|
||||
|
||||
func (s *Service) doListQuery(query backend.DataQuery) backend.DataResponse {
|
||||
func (s *Service) doListQuery(ctx context.Context, query backend.DataQuery) backend.DataResponse {
|
||||
q := &listQueryModel{}
|
||||
response := backend.DataResponse{}
|
||||
err := json.Unmarshal(query.JSON, &q)
|
||||
@@ -138,40 +107,17 @@ func (s *Service) doListQuery(query backend.DataQuery) backend.DataResponse {
|
||||
return response
|
||||
}
|
||||
|
||||
if q.Path == "" {
|
||||
count := len(s.roots)
|
||||
names := data.NewFieldFromFieldType(data.FieldTypeString, count)
|
||||
mtype := data.NewFieldFromFieldType(data.FieldTypeString, count)
|
||||
names.Name = "name"
|
||||
mtype.Name = "mediaType"
|
||||
for i, f := range s.roots {
|
||||
names.Set(i, f)
|
||||
mtype.Set(i, "directory")
|
||||
}
|
||||
frame := data.NewFrame("", names, mtype)
|
||||
frame.SetMeta(&data.FrameMeta{
|
||||
Type: data.FrameTypeDirectoryListing,
|
||||
})
|
||||
response.Frames = data.Frames{frame}
|
||||
} else {
|
||||
path, err := s.publicPath(q.Path)
|
||||
if err != nil {
|
||||
response.Error = err
|
||||
return response
|
||||
}
|
||||
frame, err := experimental.GetDirectoryFrame(path, false)
|
||||
if err != nil {
|
||||
response.Error = err
|
||||
return response
|
||||
}
|
||||
path := store.RootPublicStatic + "/" + q.Path
|
||||
frame, err := s.store.List(ctx, nil, path)
|
||||
response.Error = err
|
||||
if frame != nil {
|
||||
response.Frames = data.Frames{frame}
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
func (s *Service) doReadQuery(query backend.DataQuery) backend.DataResponse {
|
||||
q := &listQueryModel{}
|
||||
func (s *Service) doReadQuery(ctx context.Context, query backend.DataQuery) backend.DataResponse {
|
||||
q := &readQueryModel{}
|
||||
response := backend.DataResponse{}
|
||||
err := json.Unmarshal(query.JSON, &q)
|
||||
if err != nil {
|
||||
@@ -184,27 +130,14 @@ func (s *Service) doReadQuery(query backend.DataQuery) backend.DataResponse {
|
||||
return response
|
||||
}
|
||||
|
||||
path, err := s.publicPath(q.Path)
|
||||
path := store.RootPublicStatic + "/" + q.Path
|
||||
file, err := s.store.Read(ctx, nil, path)
|
||||
if err != nil {
|
||||
response.Error = err
|
||||
return response
|
||||
}
|
||||
|
||||
// Can ignore gosec G304 here, because we check the file pattern above
|
||||
// nolint:gosec
|
||||
fileReader, err := os.Open(path)
|
||||
if err != nil {
|
||||
response.Error = fmt.Errorf("failed to read file")
|
||||
return response
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := fileReader.Close(); err != nil {
|
||||
logger.Warn("Failed to close file", "err", err, "path", path)
|
||||
}
|
||||
}()
|
||||
|
||||
frame, err := testdatasource.LoadCsvContent(fileReader, filepath.Base(path))
|
||||
frame, err := testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), filepath.Base(path))
|
||||
if err != nil {
|
||||
response.Error = err
|
||||
return response
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
package grafanads
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/searchV2"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend"
|
||||
"github.com/grafana/grafana-plugin-sdk-go/experimental"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func asJSON(v interface{}) json.RawMessage {
|
||||
b, _ := json.Marshal(v)
|
||||
return b
|
||||
}
|
||||
|
||||
func TestReadFolderListing(t *testing.T) {
|
||||
ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, searchV2.NewStubSearchService())
|
||||
dr := ds.doListQuery(backend.DataQuery{
|
||||
QueryType: "x",
|
||||
JSON: asJSON(listQueryModel{
|
||||
Path: "testdata",
|
||||
}),
|
||||
})
|
||||
err := experimental.CheckGoldenDataResponse(path.Join("testdata", "list.golden.txt"), &dr, true)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestReadCSVFile(t *testing.T) {
|
||||
ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, searchV2.NewStubSearchService())
|
||||
dr := ds.doReadQuery(backend.DataQuery{
|
||||
QueryType: "x",
|
||||
JSON: asJSON(readQueryModel{
|
||||
Path: "testdata/js_libraries.csv",
|
||||
}),
|
||||
})
|
||||
err := experimental.CheckGoldenDataResponse(path.Join("testdata", "jslib.golden.txt"), &dr, true)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
-21
@@ -1,21 +0,0 @@
|
||||
🌟 This was machine generated. Do not edit. 🌟
|
||||
|
||||
Frame[0]
|
||||
Name: js_libraries.csv
|
||||
Dimensions: 4 Fields by 6 Rows
|
||||
+-----------------+--------------------+----------------+----------------+
|
||||
| Name: Library | Name: Github Stars | Name: Forks | Name: Watchers |
|
||||
| Labels: | Labels: | Labels: | Labels: |
|
||||
| Type: []*string | Type: []*int64 | Type: []*int64 | Type: []*int64 |
|
||||
+-----------------+--------------------+----------------+----------------+
|
||||
| React.js | 169000 | 34000 | 6700 |
|
||||
| Vue | 184000 | 29100 | 6300 |
|
||||
| Angular | 73400 | 19300 | 3200 |
|
||||
| JQuery | 54900 | 20000 | 3300 |
|
||||
| Meteor | 42400 | 5200 | 1700 |
|
||||
| Aurelia | 11600 | 684 | 442 |
|
||||
+-----------------+--------------------+----------------+----------------+
|
||||
|
||||
|
||||
====== TEST DATA RESPONSE (arrow base64) ======
|
||||
FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAGAAAAACAAAAKAAAAAQAAAAs/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAEz+//8IAAAAHAAAABAAAABqc19saWJyYXJpZXMuY3N2AAAAAAQAAABuYW1lAAAAAAQAAABgAQAA1AAAAHAAAAAEAAAAwv7//xQAAABAAAAAQAAAAAAAAgFEAAAAAQAAAAQAAACw/v//CAAAABQAAAAIAAAAV2F0Y2hlcnMAAAAABAAAAG5hbWUAAAAAAAAAADT///8AAAABQAAAAAgAAABXYXRjaGVycwAAAAAq////FAAAADwAAAA8AAAAAAACAUAAAAABAAAABAAAABj///8IAAAAEAAAAAUAAABGb3JrcwAAAAQAAABuYW1lAAAAAAAAAACY////AAAAAUAAAAAFAAAARm9ya3MAAACK////FAAAAEQAAABMAAAAAAACAVAAAAABAAAABAAAAHj///8IAAAAGAAAAAwAAABHaXRodWIgU3RhcnMAAAAABAAAAG5hbWUAAAAAAAAAAAgADAAIAAcACAAAAAAAAAFAAAAADAAAAEdpdGh1YiBTdGFycwAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAABQFEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAcAAABMaWJyYXJ5AAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAcAAABMaWJyYXJ5AP////8oAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA2AAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAqAAAAAYAAAAAAAAAAAAAAAkAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAKAAAAAAAAABIAAAAAAAAAAAAAAAAAAAASAAAAAAAAAAwAAAAAAAAAHgAAAAAAAAAAAAAAAAAAAB4AAAAAAAAADAAAAAAAAAAqAAAAAAAAAAAAAAAAAAAAKgAAAAAAAAAMAAAAAAAAAAAAAAABAAAAAYAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAYAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAsAAAASAAAAGAAAAB4AAAAlAAAAAAAAAFJlYWN0LmpzVnVlQW5ndWxhckpRdWVyeU1ldGVvckF1cmVsaWEAAAAolAIAAAAAAMDOAgAAAAAAuB4BAAAAAAB01gAAAAAAAKClAAAAAAAAUC0AAAAAAADQhAAAAAAAAKxxAAAAAAAAZEsAAAAAAAAgTgAAAAAAAFAUAAAAAAAArAIAAAAAAAAsGgAAAAAAAJwYAAAAAAAAgAwAAAAAAADkDAAAAAAAAKQGAAAAAAAAugEAAAAAAAAQAAAADAAUABIADAAIAAQADAAAABAAAAAsAAAAPAAAAAAAAwABAAAAaAIAAAAAAAAwAQAAAAAAANgAAAAAAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABgAAAAAgAAACgAAAAEAAAALP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABM/v//CAAAABwAAAAQAAAAanNfbGlicmFyaWVzLmNzdgAAAAAEAAAAbmFtZQAAAAAEAAAAYAEAANQAAABwAAAABAAAAML+//8UAAAAQAAAAEAAAAAAAAIBRAAAAAEAAAAEAAAAsP7//wgAAAAUAAAACAAAAFdhdGNoZXJzAAAAAAQAAABuYW1lAAAAAAAAAAA0////AAAAAUAAAAAIAAAAV2F0Y2hlcnMAAAAAKv///xQAAAA8AAAAPAAAAAAAAgFAAAAAAQAAAAQAAAAY////CAAAABAAAAAFAAAARm9ya3MAAAAEAAAAbmFtZQAAAAAAAAAAmP///wAAAAFAAAAABQAAAEZvcmtzAAAAiv///xQAAABEAAAATAAAAAAAAgFQAAAAAQAAAAQAAAB4////CAAAABgAAAAMAAAAR2l0aHViIFN0YXJzAAAAAAQAAABuYW1lAAAAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAwAAABHaXRodWIgU3RhcnMAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAUBRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAHAAAATGlicmFyeQAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAHAAAATGlicmFyeQCIAgAAQVJST1cx
|
||||
-25
@@ -1,25 +0,0 @@
|
||||
🌟 This was machine generated. Do not edit. 🌟
|
||||
|
||||
Frame[0] {
|
||||
"type": "directory-listing",
|
||||
"pathSeparator": "/"
|
||||
}
|
||||
Name:
|
||||
Dimensions: 2 Fields by 7 Rows
|
||||
+--------------------------+------------------+
|
||||
| Name: name | Name: media-type |
|
||||
| Labels: | Labels: |
|
||||
| Type: []string | Type: []string |
|
||||
+--------------------------+------------------+
|
||||
| browser_marketshare.csv | |
|
||||
| flight_info_by_state.csv | |
|
||||
| gdp_per_capita.csv | |
|
||||
| js_libraries.csv | |
|
||||
| ohlc_dogecoin.csv | |
|
||||
| population_by_state.csv | |
|
||||
| weight_height.csv | |
|
||||
+--------------------------+------------------+
|
||||
|
||||
|
||||
====== TEST DATA RESPONSE (arrow base64) ======
|
||||
FRAME=QVJST1cxAAD/////uAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAKQAAAADAAAATAAAACgAAAAEAAAA0P7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADw/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAABD///8IAAAAPAAAADAAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInBhdGhTZXBhcmF0b3IiOiIvIn0AAAAABAAAAG1ldGEAAAAAAgAAAHwAAAAEAAAAnv///xQAAABAAAAAQAAAAAAAAAU8AAAAAQAAAAQAAACM////CAAAABQAAAAKAAAAbWVkaWEtdHlwZQAABAAAAG5hbWUAAAAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA/////9gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAADQAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAAB4AAAABwAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAACQAAAAAAAAALAAAAAAAAAAAAAAAAAAAACwAAAAAAAAACAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAAAAACAAAABwAAAAAAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAFwAAAC8AAABBAAAAUQAAAGIAAAB5AAAAigAAAGJyb3dzZXJfbWFya2V0c2hhcmUuY3N2ZmxpZ2h0X2luZm9fYnlfc3RhdGUuY3N2Z2RwX3Blcl9jYXBpdGEuY3N2anNfbGlicmFyaWVzLmNzdm9obGNfZG9nZWNvaW4uY3N2cG9wdWxhdGlvbl9ieV9zdGF0ZS5jc3Z3ZWlnaHRfaGVpZ2h0LmNzdgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAMAAQAAAMgBAAAAAAAA4AAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAApAAAAAMAAABMAAAAKAAAAAQAAADQ/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAPD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAEP///wgAAAA8AAAAMAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAfAAAAAQAAACe////FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAAIz///8IAAAAFAAAAAoAAABtZWRpYS10eXBlAAAEAAAAbmFtZQAAAAAAAAAAiP///woAAABtZWRpYS10eXBlAAAAABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAAFRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAbmFtZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAEAAAAbmFtZQAAAADoAQAAQVJST1cx
|
||||
Reference in New Issue
Block a user