public/grafana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Elasticsearch: Use _field_caps instead of _mapping to get fields (#97607)

Browse Source
This commit is contained in:
Isabella Siu
2024-12-12 17:20:04 -05:00
committed by GitHub
parent a362769491
commit b3a12f486e
10 changed files with 296 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md
+1
View File
@@ -120,6 +120,7 @@ Most [generally available](https://grafana.com/docs/release-life-cycle/#general-
| `ssoSettingsLDAP` | Use the new SSO Settings API to configure LDAP |
| `useSessionStorageForRedirection` | Use session storage for handling the redirection after login |
| `reportingUseRawTimeRange` | Uses the original report or dashboard time range instead of making an absolute transformation |
| `elasticsearchCrossClusterSearch` | Enables cross cluster search in the Elasticsearch datasource |
## Experimental feature toggles
packages/grafana-data/src/types/featureToggles.gen.ts
+1
View File
@@ -242,4 +242,5 @@ export interface FeatureToggles {
azureMonitorEnableUserAuth?: boolean;
alertingNotificationsStepMode?: boolean;
feedbackButton?: boolean;
elasticsearchCrossClusterSearch?: boolean;
}
pkg/services/featuremgmt/registry.go
+6
View File
@@ -1676,6 +1676,12 @@ var (
Owner: grafanaOperatorExperienceSquad,
HideFromDocs: true,
},
{
Name: "elasticsearchCrossClusterSearch",
Description: "Enables cross cluster search in the Elasticsearch datasource",
Stage: FeatureStagePublicPreview,
Owner: awsDatasourcesSquad,
},
}
)
pkg/services/featuremgmt/toggles_gen.csv
+1
View File
@@ -223,3 +223,4 @@ alertingUIOptimizeReducer,GA,@grafana/alerting-squad,false,false,true
azureMonitorEnableUserAuth,GA,@grafana/partner-datasources,false,false,false
alertingNotificationsStepMode,experimental,@grafana/alerting-squad,false,false,true
feedbackButton,experimental,@grafana/grafana-operator-experience-squad,false,false,false
elasticsearchCrossClusterSearch,preview,@grafana/aws-datasources,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
223 azureMonitorEnableUserAuth GA @grafana/partner-datasources false false false
224 alertingNotificationsStepMode experimental @grafana/alerting-squad false false true
225 feedbackButton experimental @grafana/grafana-operator-experience-squad false false false
226 elasticsearchCrossClusterSearch preview @grafana/aws-datasources false false false
pkg/services/featuremgmt/toggles_gen.go
+4
View File
@@ -902,4 +902,8 @@ const (
// FlagFeedbackButton
// Enables a button to send feedback from the Grafana UI
FlagFeedbackButton = "feedbackButton"
// FlagElasticsearchCrossClusterSearch
// Enables cross cluster search in the Elasticsearch datasource
FlagElasticsearchCrossClusterSearch = "elasticsearchCrossClusterSearch"
)
pkg/services/featuremgmt/toggles_gen.json
+15
View File
@@ -1269,6 +1269,21 @@
"frontend": true
}
},
{
"metadata": {
"name": "elasticsearchCrossClusterSearch",
"resourceVersion": "1733848475752",
"creationTimestamp": "2024-12-09T13:53:38Z",
"annotations": {
"grafana.app/updatedTimestamp": "2024-12-10 16:34:35.752111 +0000 UTC"
}
},
"spec": {
"description": "Enables cross cluster search in the Elasticsearch datasource",
"stage": "preview",
"codeowner": "@grafana/aws-datasources"
}
},
{
"metadata": {
"name": "enableDatagridEditing",
pkg/tsdb/elasticsearch/elasticsearch.go
+11 -1
View File
@@ -182,14 +182,21 @@ func (s *Service) getDSInfo(ctx context.Context, pluginCtx backend.PluginContext
return &instance, nil
}
func isFieldCaps(url string) bool {
return strings.HasSuffix(url, "/_field_caps") || url == "_field_caps"
}
func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
logger := s.logger.FromContext(ctx)
// allowed paths for resource calls:
// - empty string for fetching db version
// - /_mapping for fetching index mapping, e.g. requests going to `index/_mapping`
// - /_field_caps for fetching field capabilities, e.g. requests going to `index/_field_caps`
// - _msearch for executing getTerms queries
// - _mapping for fetching "root" index mappings
if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_msearch" && req.Path != "_mapping" {
// - _field_caps for fetching "root" field capabilities
if req.Path != "" && !isFieldCaps(req.Path) && req.Path != "_msearch" &&
!strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_mapping" {
logger.Error("Invalid resource path", "path", req.Path)
return fmt.Errorf("invalid resource URL: %s", req.Path)
}
@@ -266,6 +273,9 @@ func createElasticsearchURL(req *backend.CallResourceRequest, ds *es.DatasourceI
}
esUrl.Path = path.Join(esUrl.Path, req.Path)
if isFieldCaps(req.Path) {
esUrl.RawQuery = "fields=*"
}
esUrlString := esUrl.String()
// If the request path is empty and the URL does not end with a slash, add a slash to the URL.
// This ensures that for version checks executed to the root URL, the URL ends with a slash.
public/app/plugins/datasource/elasticsearch/datasource.test.ts
+188 -1
View File
@@ -12,7 +12,7 @@ import {
TimeRange,
toUtc,
} from '@grafana/data';
import { FetchResponse, reportInteraction, getBackendSrv, setBackendSrv, BackendSrv } from '@grafana/runtime';
import { FetchResponse, reportInteraction, getBackendSrv, setBackendSrv, BackendSrv, config } from '@grafana/runtime';
import { ElasticDatasource } from './datasource';
import { createElasticDatasource, createElasticQuery, mockResponseFrames } from './mocks';
@@ -1253,4 +1253,191 @@ describe('ElasticDatasource', () => {
});
});
});
describe('getFieldsFieldCap', () => {
const originalFeatureToggleValue = config.featureToggles.elasticsearchCrossClusterSearch;
afterEach(() => {
config.featureToggles.elasticsearchCrossClusterSearch = originalFeatureToggleValue;
});
const getFieldsMockData = {
fields: {
'@timestamp_millis': {
date: {
type: 'date',
metadata_field: false,
},
},
classification_terms: {
keyword: {
type: 'keyword',
metadata_field: false,
},
},
ip_address: {
ip: {
type: 'ip',
metadata_field: false,
},
},
'justification_blob.criterion.keyword': {
keyword: {
type: 'keyword',
metadata_field: false,
},
},
'justification_blob.criterion': {
text: {
type: 'text',
metadata_field: false,
},
},
justification_blob: {
object: {
type: 'object',
metadata_field: false,
},
},
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness': {
float: {
type: 'float',
metadata_field: false,
},
},
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score': {
float: {
type: 'float',
metadata_field: false,
},
},
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness': {
float: {
type: 'float',
metadata_field: false,
},
},
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score': {
float: {
type: 'float',
metadata_field: false,
},
},
overall_vote_score: {
float: {
type: 'float',
metadata_field: false,
},
},
_index: {
_index: {
type: '_index',
metadata_field: true,
},
},
},
indices: ['[test-]YYYY.MM.DD'],
};
it('should not retry when ES is down', async () => {
config.featureToggles.elasticsearchCrossClusterSearch = true;
const twoDaysBefore = toUtc().subtract(2, 'day').format('YYYY.MM.DD');
const ds = createElasticDatasource({
jsonData: { interval: 'Daily' },
});
ds.getResource = jest.fn().mockImplementation((options) => {
if (options.url === `test-${twoDaysBefore}/_field_caps`) {
return of({
data: {},
});
}
return throwError({ status: 500 });
});
const timeRange = { from: 1, to: 2 } as unknown as TimeRange;
await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
expect(received[0]).toStrictEqual({ status: 500 });
expect(ds.getResource).toBeCalledTimes(1);
});
});
it('should not retry more than 7 indices', async () => {
config.featureToggles.elasticsearchCrossClusterSearch = true;
const ds = createElasticDatasource({
jsonData: { interval: 'Daily' },
});
ds.getResource = jest.fn().mockImplementation(() => {
return throwError({ status: 404 });
});
const timeRange = createTimeRange(dateTime().subtract(2, 'week'), dateTime());
await expect(ds.getFields(undefined, timeRange)).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
expect(received[0]).toStrictEqual('Could not find an available index for this time range.');
expect(ds.getResource).toBeCalledTimes(7);
});
});
it('should return nested fields', async () => {
config.featureToggles.elasticsearchCrossClusterSearch = true;
const ds = createElasticDatasource({
jsonData: { interval: 'Daily' },
});
ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
await expect(ds.getFields()).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
const fieldObjects = received[0];
const fields = map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp_millis',
'classification_terms',
'ip_address',
'justification_blob.criterion.keyword',
'justification_blob.criterion',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
]);
});
});
it('should return number fields', async () => {
config.featureToggles.elasticsearchCrossClusterSearch = true;
ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
await expect(ds.getFields(['number'])).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
const fieldObjects = received[0];
const fields = map(fieldObjects, 'text');
expect(fields).toEqual([
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
]);
});
});
it('should return date fields', async () => {
config.featureToggles.elasticsearchCrossClusterSearch = true;
ds.getResource = jest.fn().mockResolvedValue(getFieldsMockData);
await expect(ds.getFields(['date'])).toEmitValuesWith((received) => {
expect(received.length).toBe(1);
const fieldObjects = received[0];
const fields = map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp_millis']);
});
});
});
});
public/app/plugins/datasource/elasticsearch/datasource.ts
+68 -2
View File
@@ -1,4 +1,4 @@
import { cloneDeep, first as _first, isNumber, isObject, isString, map as _map, find } from 'lodash';
import { cloneDeep, first as _first, isNumber, isString, map as _map, find, isObject } from 'lodash';
import { from, generate, lastValueFrom, Observable, of } from 'rxjs';
import { catchError, first, map, mergeMap, skipWhile, throwIfEmpty, tap } from 'rxjs/operators';
import { SemVer } from 'semver';
@@ -46,6 +46,7 @@ import {
BackendSrvRequest,
TemplateSrv,
getTemplateSrv,
config,
} from '@grafana/runtime';
import { IndexPattern, intervalMap } from './IndexPattern';
@@ -197,7 +198,7 @@ export class ElasticDatasource
indexList = [this.indexPattern.getIndexForToday()];
}
const url = '_mapping';
const url = config.featureToggles.elasticsearchCrossClusterSearch ? '_field_caps' : '_mapping';
const indexUrlList = indexList.map((index) => {
// make sure `index` does not end with a slash
@@ -743,6 +744,10 @@ export class ElasticDatasource
* or fix the implementation.
*/
getFields(type?: string[], range?: TimeRange): Observable<MetricFindValue[]> {
if (config.featureToggles.elasticsearchCrossClusterSearch) {
return this.getFieldsCrossCluster(type, range);
}
const typeMap: Record<string, string> = {
float: 'number',
double: 'number',
@@ -823,6 +828,67 @@ export class ElasticDatasource
);
}
getFieldsCrossCluster(type?: string[], range?: TimeRange): Observable<MetricFindValue[]> {
const typeMap: Record<string, string> = {
float: 'number',
double: 'number',
integer: 'number',
long: 'number',
date: 'date',
date_nanos: 'date',
string: 'string',
text: 'string',
scaled_float: 'number',
nested: 'nested',
histogram: 'number',
};
return this.requestAllIndices(range).pipe(
map((result) => {
interface FieldInfo {
metadata_field: string;
}
const shouldAddField = (obj: Record<string, Record<string, FieldInfo>>) => {
// equal query type filter, or via type map translation
for (const objField in obj) {
if (objField === 'object') {
continue;
}
if (obj[objField].metadata_field) {
continue;
}
if (!type || type.length === 0) {
return true;
}
if (type.includes(objField) || type.includes(typeMap[objField])) {
return true;
}
}
return false;
};
const fields: Record<string, { text: string; type: string }> = {};
const fieldsData = result['fields'];
for (const fieldName in fieldsData) {
const fieldInfo = fieldsData[fieldName];
if (shouldAddField(fieldInfo)) {
fields[fieldName] = {
text: fieldName,
type: fieldInfo.type,
};
}
}
// transform to array
return _map(fields, (value) => {
return value;
});
})
);
}
/**
* Get values for a given field.
* Used for example in getTagValues.
public/app/plugins/datasource/elasticsearch/hooks/useFields.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export const useFields = (type: AggregationType | string[]) => {
let rawFields: MetricFindValue[];
return async (q?: string) => {
// _mapping doesn't support filtering, we avoid sending a request everytime q changes
// TODO: use _field_caps to support filtering
if (!rawFields) {
rawFields = await lastValueFrom(datasource.getFields(filter, range));
}