public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Storage: enable SQL backend (#48095)

Browse Source 
* #45498: add String util to ListResponse for better UX

* #45498: refactor db_filestorage FS API backend - use path_hash in DB schema

* #45498: enable DB backend fs api tests

* #45498: add comment

* #45498: enable Storage feature flag during integration tests

* remove fmt.println

* #45498: reduce sizes of hash columns

* separate conditions

* #45498: make it easy to ignore backends when running fs api integration tests

* #45498: quote `key` column name

* #45498: reduce path_hash size

* #45498: verify `/{orgId}/{storageName}/` prefix convention in integration tests

* #45498: add etag to the sql table

* #45498: add etag to the sql table

* remove feature flag check (storage isn't dev-mode only)

* add cacheControl and content disposition

* add comments

* add path_hash comment

* explicitly set `path` column collation in `file` table for postgres
This commit is contained in:
Artur Wierzbicki
2022-05-16 10:26:40 -07:00
committed by GitHub
parent ff844f0599
commit 5c321599c8
6 changed files with 274 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/migrations/db_file_storage.go
+33 -7
View File
@@ -2,40 +2,66 @@ package migrations
import "github.com/grafana/grafana/pkg/services/sqlstore/migrator"
// TODO: remove nolint as part of https://github.com/grafana/grafana/issues/45498
// nolint:unused,deadcode
func addDbFileStorageMigration(mg *migrator.Migrator) {
filesTable := migrator.Table{
Name: "file",
Columns: []*migrator.Column{
{Name: "path", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
{Name: "parent_folder_path", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
// path_hash is used for indexing. we are using it to circumvent the max length limit of 191 for varchar2 fields in MySQL 5.6
{Name: "path_hash", Type: migrator.DB_NVarchar, Length: 64, Nullable: false},
// parent_folder_path_hash is an optimization for a common use case - list all files in a given folder
{Name: "parent_folder_path_hash", Type: migrator.DB_NVarchar, Length: 64, Nullable: false},
{Name: "contents", Type: migrator.DB_Blob, Nullable: false},
// HTTP Entity tag; md5 hash
{Name: "etag", Type: migrator.DB_NVarchar, Length: 32, Nullable: false},
// cache_control HTTP header
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control
{Name: "cache_control", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
// content_disposition HTTP header - inline/attachment file display
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
{Name: "content_disposition", Type: migrator.DB_NVarchar, Length: 128, Nullable: false},
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false},
{Name: "created", Type: migrator.DB_DateTime, Nullable: false},
{Name: "size", Type: migrator.DB_BigInt, Nullable: false},
{Name: "mime_type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false},
},
Indices: []*migrator.Index{
{Cols: []string{"path"}, Type: migrator.UniqueIndex},
{Cols: []string{"path_hash"}, Type: migrator.UniqueIndex},
{Cols: []string{"parent_folder_path_hash"}},
},
}
mg.AddMigration("create file table", migrator.NewAddTableMigration(filesTable))
mg.AddMigration("file table idx: path natural pk", migrator.NewAddIndexMigration(filesTable, filesTable.Indices[0]))
mg.AddMigration("file table idx: parent_folder_path_hash fast folder retrieval", migrator.NewAddIndexMigration(filesTable, filesTable.Indices[1]))
fileMetaTable := migrator.Table{
Name: "file_meta",
Columns: []*migrator.Column{
{Name: "path", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
{Name: "key", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
{Name: "path_hash", Type: migrator.DB_NVarchar, Length: 64, Nullable: false},
// 191 is the maximum length of indexable VARCHAR fields in MySQL 5.6 <= with utf8mb4 encoding
{Name: "key", Type: migrator.DB_NVarchar, Length: 191, Nullable: false},
{Name: "value", Type: migrator.DB_NVarchar, Length: 1024, Nullable: false},
},
Indices: []*migrator.Index{
{Cols: []string{"path", "key"}, Type: migrator.UniqueIndex},
{Cols: []string{"path_hash", "key"}, Type: migrator.UniqueIndex},
},
}
mg.AddMigration("create file_meta table", migrator.NewAddTableMigration(fileMetaTable))
mg.AddMigration("file table idx: path key", migrator.NewAddIndexMigration(fileMetaTable, fileMetaTable.Indices[0]))
// TODO: add collation support to `migrator.Column`
mg.AddMigration("set path collation in file table", migrator.NewRawSQLMigration("").
// MySQL `utf8mb4_unicode_ci` collation is set in `mysql_dialect.go`
// SQLite uses a `BINARY` collation by default
Postgres("ALTER TABLE file ALTER COLUMN path TYPE VARCHAR(1024) COLLATE \"C\";")) // Collate C - sorting done based on character code byte values
}
pkg/services/sqlstore/migrations/migrations.go
+1
View File
@@ -95,6 +95,7 @@ func (*OSSMigrations) AddMigration(mg *Migrator) {
addPublicDashboardMigration(mg)
ualert.CreateDefaultFoldersForAlertingMigration(mg)
addDbFileStorageMigration(mg)
}
func addMigrationLogMigrations(mg *Migrator) {