public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Storage: Unified Storage based on Entity API (#71977)

Browse Source 
* first round of entityapi updates

- quote column names and clean up insert/update queries
- replace grn with guid
- streamline table structure

fixes

streamline entity history

move EntitySummary into proto

remove EntitySummary

add guid to json

fix tests

change DB_Uuid to DB_NVarchar

fix folder test

convert interface to any

more cleanup

start entity store under grafana-apiserver dskit target

CRUD working, kind of

rough cut of wiring entity api to kube-apiserver

fake grafana user in context

add key to entity

list working

revert unnecessary changes

move entity storage files to their own package, clean up

use accessor to read/write grafana annotations

implement separate Create and Update functions

* go mod tidy

* switch from Kind to resource

* basic grpc storage server

* basic support for grpc entity store

* don't connect to database unless it's needed, pass user identity over grpc

* support getting user from k8s context, fix some mysql issues

* assign owner to snowflake dependency

* switch from ulid to uuid for guids

* cleanup, rename Search to List

* remove entityListResult

* EntityAPI: remove extra user abstraction (#79033)

* remove extra user abstraction

* add test stub (but

* move grpc context setup into client wrapper, fix lint issue

* remove unused constants

* remove custom json stuff

* basic list filtering, add todo

* change target to storage-server, allow entityStore flag in prod mode

* fix issue with Update

* EntityAPI: make test work, need to resolve expected differences (#79123)

* make test work, need to resolve expected differences

* remove the fields not supported by legacy

* sanitize out the bits legacy does not support

* sanitize out the bits legacy does not support

---------

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

* update feature toggle generated files

* remove unused http headers

* update feature flag strategy

* devmode

* update readme

* spelling

* readme

---------

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
This commit is contained in:
Dan Cech
2023-12-06 15:21:21 -05:00
committed by GitHub
co-authored by Ryan McKinley
parent 07915703fe
commit c4c9bfaf2e
42 changed files with 4357 additions and 2388 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/store/entity/entity.proto
+205 -188
View File
@@ -7,41 +7,75 @@ import "pkg/infra/grn/grn.proto";
// The canonical entity/document data -- this represents the raw bytes and storage level metadata
message Entity {
// Entity identifier -- tenant_id, kind, uid
grn.GRN GRN = 1;
// The version will change when the entity is saved. It is not necessarily sortable
// Globally unique ID set by the system. This can not be set explicitly
string guid = 1;
// The resourceVersion -- it will change whenever anythign on the object is saved
string version = 2;
// Time in epoch milliseconds that the entity was created
int64 created_at = 3;
// Entity identifier -- tenant_id, kind, uid
grn.GRN GRN = 3;
// Time in epoch milliseconds that the entity was updated
int64 updated_at = 4;
// group version of the entity
string group_version = 23;
// Who created the entity
string created_by = 5;
// k8s key value
string key = 22;
// Who updated the entity
string updated_by = 6;
// The folder UID
string folder = 4;
// The folder UID (not stored in the body)
string folder = 7;
// Raw meta from k8s
bytes meta = 5;
// MD5 digest of the body
string ETag = 8;
// Raw bytes of the storage entity. The kind will determine what is a valid payload
bytes body = 6;
// k8s style status (ignored for now)
bytes status = 7;
// the friendly name of the entity
string name = 8;
// Content Length
int64 size = 9;
// Raw bytes of the storage entity. The kind will determine what is a valid payload
bytes body = 10;
// MD5 digest of the body
string ETag = 10;
// Entity summary as JSON
bytes summary_json = 11;
// Time in epoch milliseconds that the entity was created
int64 created_at = 11;
// Who created the entity
string created_by = 12;
// Time in epoch milliseconds that the entity was updated
int64 updated_at = 13;
// Who updated the entity
string updated_by = 14;
// External location info
EntityOriginInfo origin = 12;
EntityOriginInfo origin = 15;
// human-readable description of the entity
string description = 16;
// URL safe version of the name. It will be unique within the folder
string slug = 17;
// Commit message (optional)
string message = 18;
// Key value pairs. Tags are are represented as keys with empty values
map<string,string> labels = 19;
// Optional field values. The schema will define and document possible values for a given kind
map<string, string> fields = 20;
// When errors exist
repeated EntityErrorInfo errors = 21;
}
// This stores additional metadata for items entities that were synced from external systmes
@@ -69,29 +103,6 @@ message EntityErrorInfo {
bytes details_json = 3;
}
// This is a subset of Entity that does not include body or sync info
message EntityVersionInfo {
// The version will change when the entity is saved. It is not necessarily sortable
string version = 1;
// Time in epoch milliseconds that the entity was updated
int64 updated_at = 2;
// Who updated the entity
string updated_by = 3;
// Content Length
int64 size = 4;
// MD5 digest of the body
string ETag = 5;
// optional "save" or "commit" message
//
// NOTE: currently managed by the dashboard_version table, and will be returned from a "history" command
string comment = 6;
}
//-----------------------------------------------
// Get request/response
//-----------------------------------------------
@@ -100,14 +111,22 @@ message ReadEntityRequest {
// Entity identifier
grn.GRN GRN = 1;
// Fetch an explicit version
string key = 7;
// Fetch an explicit version (default is latest)
string version = 2;
// Include the full meta bytes
bool with_meta = 3;
// Include the full body bytes
bool with_body = 3;
bool with_body = 4;
// Include the status bytes (ignored for now)
bool with_status = 5;
// Include derived summary metadata
bool with_summary = 4;
bool with_summary = 6;
}
//------------------------------------------------------
@@ -127,85 +146,38 @@ message BatchReadEntityResponse {
//-----------------------------------------------
message WriteEntityRequest {
// Entity identifier
grn.GRN GRN = 1;
// Where to save the entity (empty will leave it unchanged)
string folder = 2;
// The raw entity body
bytes body = 3;
// Message that can be seen when exploring entity history
string comment = 4;
// Entity details
Entity entity = 1;
// Used for optimistic locking. If missing, the previous version will be replaced regardless
string previous_version = 5;
string previous_version = 2;
}
// This operation is useful when syncing a resource from external sources
// that have more accurate metadata information (git, or an archive).
// This process can bypass the forced checks that
// This process can bypass the forced checks that
message AdminWriteEntityRequest {
// Entity identifier
grn.GRN GRN = 1;
// Where to save the entity (empty will leave it unchanged)
string folder = 2;
// The raw entity body
bytes body = 3;
// Message that can be seen when exploring entity history
string comment = 4;
// Time in epoch milliseconds that the entity was created
// Optional, if 0 it will use the current time
int64 created_at = 5;
// Time in epoch milliseconds that the entity was updated
// Optional, if empty it will use the current user
int64 updated_at = 6;
// Who created the entity
// Optional, if 0 it will use the current time
string created_by = 7;
// Who updated the entity
// Optional, if empty it will use the current user
string updated_by = 8;
// An explicit version identifier
// Optional, if set, this will overwrite/define an explicit version
string version = 9;
// Entity details
Entity entity = 1;
// Used for optimistic locking. If missing, the previous version will be replaced regardless
// This may not be used along with an explicit version in the request
string previous_version = 10;
string previous_version = 2;
// Request that all previous versions are removed from the history
// This will make sense for systems that manage history explicitly externallay
bool clear_history = 11;
// Optionally define where the entity came from
EntityOriginInfo origin = 12;
bool clear_history = 3;
}
message WriteEntityResponse {
// Error info -- if exists, the save did not happen
EntityErrorInfo error = 1;
// Entity identifier
grn.GRN GRN = 2;
// Entity details with the body removed
EntityVersionInfo entity = 3;
// Entity summary as JSON
bytes summary_json = 4;
// Entity details
Entity entity = 2;
// Status code
Status status = 5;
Status status = 3;
// Status enumeration
enum Status {
@@ -216,6 +188,62 @@ message WriteEntityResponse {
}
}
//-----------------------------------------------
// Create request/response
//-----------------------------------------------
message CreateEntityRequest {
// Entity details
Entity entity = 1;
}
message CreateEntityResponse {
// Error info -- if exists, the save did not happen
EntityErrorInfo error = 1;
// Entity details
Entity entity = 2;
// Status code
Status status = 3;
// Status enumeration
enum Status {
ERROR = 0;
CREATED = 1;
}
}
//-----------------------------------------------
// Update request/response
//-----------------------------------------------
message UpdateEntityRequest {
// Entity details
Entity entity = 1;
// Used for optimistic locking. If missing, the previous version will be replaced regardless
string previous_version = 2;
}
message UpdateEntityResponse {
// Error info -- if exists, the save did not happen
EntityErrorInfo error = 1;
// Entity details
Entity entity = 2;
// Status code
Status status = 3;
// Status enumeration
enum Status {
ERROR = 0;
UPDATED = 1;
UNCHANGED = 2;
}
}
//-----------------------------------------------
// Delete request/response
//-----------------------------------------------
@@ -224,12 +252,28 @@ message DeleteEntityRequest {
// Entity identifier
grn.GRN GRN = 1;
// Used for optimistic locking. If missing, the previous version will be replaced regardless
string previous_version = 3;
string key = 3;
// Used for optimistic locking. If missing, the current version will be deleted regardless
string previous_version = 2;
}
message DeleteEntityResponse {
bool OK = 1;
// Error info -- if exists, the delete did not happen
EntityErrorInfo error = 1;
// Entity details
Entity entity = 2;
// Status code
Status status = 3;
// Status enumeration
enum Status {
ERROR = 0;
DELETED = 1;
NOTFOUND = 2;
}
}
//-----------------------------------------------
@@ -241,7 +285,7 @@ message EntityHistoryRequest {
grn.GRN GRN = 1;
// Maximum number of items to return
int64 limit = 3;
int64 limit = 3;
// Starting from the requested page
string next_page_token = 5;
@@ -252,7 +296,7 @@ message EntityHistoryResponse {
grn.GRN GRN = 1;
// Entity metadata without the raw bytes
repeated EntityVersionInfo versions = 2;
repeated Entity versions = 2;
// More results exist... pass this in the next request
string next_page_token = 3;
@@ -263,12 +307,12 @@ message EntityHistoryResponse {
// List request/response
//-----------------------------------------------
message EntitySearchRequest {
message EntityListRequest {
// Starting from the requested page (other query parameters must match!)
string next_page_token = 1;
// Maximum number of items to return
int64 limit = 2;
int64 limit = 2;
// Free text query string -- mileage may vary :)
string query = 3;
@@ -276,6 +320,9 @@ message EntitySearchRequest {
// limit to a specific kind (empty is all)
repeated string kind = 4;
// limit to a specific key
repeated string key = 11;
// Limit results to items in a specific folder
string folder = 5;
@@ -295,54 +342,22 @@ message EntitySearchRequest {
bool with_fields = 10;
}
// Search result metadata for each entity
message EntitySearchResult {
// Entity identifier
grn.GRN GRN = 1;
message ReferenceRequest {
// Starting from the requested page (other query parameters must match!)
string next_page_token = 1;
// The current version of this entity
string version = 2;
// Maximum number of items to return
int64 limit = 2;
// Content Length
int64 size = 3;
// Free text query string -- mileage may vary :)
string kind = 3;
// Time in epoch milliseconds that the entity was updated
int64 updated_at = 4;
// Who updated the entity
string updated_by = 5;
// Optionally include the full entity body
bytes body = 6;
//----------------------------------------
// Derived from body in the summary
//----------------------------------------
// Always included
string name = 7;
// Always included
string description = 8;
// The structured labels
map<string,string> labels = 9;
// Folder UID
string folder = 10;
// Slugified name
string slug = 11;
// Optionally include extracted JSON
bytes fields_json = 12;
// EntityErrorInfo in json
bytes error_json = 13;
// Free text query string -- mileage may vary :)
string uid = 4;
}
message EntitySearchResponse {
repeated EntitySearchResult results = 1;
message EntityListResponse {
repeated Entity results = 1;
// More results exist... pass this in the next request
string next_page_token = 2;
@@ -353,9 +368,9 @@ message EntitySearchResponse {
//-----------------------------------------------
message EntityWatchRequest {
// Timestamp of last changes. Empty will default to
int64 since = 1;
// Timestamp of last changes. Empty will default to
int64 since = 1;
// Watch sppecific entities
repeated grn.GRN GRN = 2;
@@ -380,7 +395,7 @@ message EntityWatchRequest {
message EntityWatchResponse {
// Timestamp the event was sent
int64 timestamp = 1;
int64 timestamp = 1;
// List of entities with the same action
repeated Entity entity = 2;
@@ -398,47 +413,47 @@ message EntityWatchResponse {
message EntitySummary {
string UID = 1;
string kind = 2;
string kind = 2;
string name = 3;
string description = 4;
string name = 3;
string description = 4;
// Key value pairs. Tags are are represented as keys with empty values
map<string,string> labels = 5;
// Key value pairs. Tags are are represented as keys with empty values
map<string,string> labels = 5;
// Parent folder UID
string folder = 6;
// Parent folder UID
string folder = 6;
// URL safe version of the name. It will be unique within the folder
string slug = 7;
// URL safe version of the name. It will be unique within the folder
string slug = 7;
// When errors exist
EntityErrorInfo error = 8;
// When errors exist
EntityErrorInfo error = 8;
// Optional field values. The schema will define and document possible values for a given kind
map<string, string> fields = 9;
// Optional field values. The schema will define and document possible values for a given kind
map<string, string> fields = 9;
// eg: panels within dashboard
repeated EntitySummary nested = 10;
// eg: panels within dashboard
repeated EntitySummary nested = 10;
// Optional references to external things
repeated EntityExternalReference references = 11;
// Optional references to external things
repeated EntityExternalReference references = 11;
}
message EntityExternalReference {
// Category of dependency
// eg: datasource, plugin, runtime
string family = 1;
// Category of dependency
// eg: datasource, plugin, runtime
string family = 1;
// datasource > prometheus|influx|...
// plugin > panel | datasource
// runtime > transformer
string type = 2;
// datasource > prometheus|influx|...
// plugin > panel | datasource
// runtime > transformer
string type = 2;
// datasource > UID
// plugin > plugin identifier
// runtime > name lookup
string identifier = 3;
// datasource > UID
// plugin > plugin identifier
// runtime > name lookup
string identifier = 3;
}
@@ -451,11 +466,13 @@ service EntityStore {
rpc Read(ReadEntityRequest) returns (Entity);
rpc BatchRead(BatchReadEntityRequest) returns (BatchReadEntityResponse);
rpc Write(WriteEntityRequest) returns (WriteEntityResponse);
rpc Create(CreateEntityRequest) returns (CreateEntityResponse);
rpc Update(UpdateEntityRequest) returns (UpdateEntityResponse);
rpc Delete(DeleteEntityRequest) returns (DeleteEntityResponse);
rpc History(EntityHistoryRequest) returns (EntityHistoryResponse);
rpc Search(EntitySearchRequest) returns (EntitySearchResponse);
rpc List(EntityListRequest) returns (EntityListResponse);
rpc Watch(EntityWatchRequest) returns (stream EntityWatchResponse);
// TEMPORARY... while we split this into a new service (see below)
rpc AdminWrite(AdminWriteEntityRequest) returns (WriteEntityResponse);
}