Graphite: Limit number of suggestions displayed in Graphite drop downs (#42056)

* Limit number of suggestions displayed in Graphite dropdowns

* Use limit API to reduce number of loaded tags for autocomplete

* Make tests more explicit

public/app/plugins/datasource/graphite/specs/store.test.ts

@@ -3,7 +3,12 @@ import gfunc from '../gfunc';
 import { TemplateSrvStub } from 'test/specs/helpers';
 import { silenceConsoleOutput } from 'test/core/utils/silenceConsoleOutput';
 import { actions } from '../state/actions';
-import { getAltSegmentsSelectables, getTagsSelectables, getTagsAsSegmentsSelectables } from '../state/providers';
+import {
+  getAltSegmentsSelectables,
+  getTagsSelectables,
+  getTagsAsSegmentsSelectables,
+  getTagValuesSelectables,
+} from '../state/providers';
 import { GraphiteSegment } from '../types';
 import { createStore } from '../state/store';
 
@@ -37,6 +42,7 @@ describe('Graphite actions', async () => {
       waitForFuncDefsLoaded: jest.fn(() => Promise.resolve(null)),
       createFuncInstance: gfunc.createFuncInstance,
       getTagsAutoComplete: jest.fn().mockReturnValue(Promise.resolve([])),
+      getTagValuesAutoComplete: jest.fn().mockReturnValue(Promise.resolve([])),
     },
     target: { target: 'aliasByNode(scaleToSeconds(test.prod.*,1),2)' },
   } as any;
@@ -210,18 +216,23 @@ describe('Graphite actions', async () => {
     });
   });
 
-  it('current time range is passed when getting list of tags when editing', async () => {
+  it('current time range and limit is passed when getting list of tags when editing', async () => {
     const currentRange = { from: 0, to: 1 };
     ctx.state.range = currentRange;
     await getTagsSelectables(ctx.state, 0, 'any');
-    expect(ctx.state.datasource.getTagsAutoComplete).toBeCalledWith([], 'any', { range: currentRange });
+    expect(ctx.state.datasource.getTagsAutoComplete).toBeCalledWith([], 'any', { range: currentRange, limit: 5000 });
   });
 
-  it('current time range is passed when getting list of tags for adding', async () => {
+  it('current time range and limit is passed when getting list of tags for adding', async () => {
     const currentRange = { from: 0, to: 1 };
     ctx.state.range = currentRange;
     await getTagsAsSegmentsSelectables(ctx.state, 'any');
-    expect(ctx.state.datasource.getTagsAutoComplete).toBeCalledWith([], 'any', { range: currentRange });
+    expect(ctx.state.datasource.getTagsAutoComplete).toBeCalledWith([], 'any', { range: currentRange, limit: 5000 });
+  });
+
+  it('limit is passed when getting list of tag values', async () => {
+    await getTagValuesSelectables(ctx.state, { key: 'key', operator: '=', value: 'value' }, 1, 'test');
+    expect(ctx.state.datasource.getTagValuesAutoComplete).toBeCalledWith([], 'key', 'test', { limit: 5000 });
   });
 
   describe('when autocomplete for metric names is not available', () => {
@@ -472,4 +483,46 @@ describe('Graphite actions', async () => {
       expect(ctx.state.target.target).toEqual(expected);
     });
   });
+
+  describe('when auto-completing over a large set of tags and metrics', () => {
+    const manyMetrics: Array<{ text: string }> = [],
+      max = 20000;
+
+    beforeEach(() => {
+      for (let i = 0; i < max; i++) {
+        manyMetrics.push({ text: `metric${i}` });
+      }
+      ctx.state.datasource.metricFindQuery = jest.fn().mockReturnValue(Promise.resolve(manyMetrics));
+      ctx.state.datasource.getTagsAutoComplete = jest.fn((_tag, _prefix, { limit }) => {
+        const tags = [];
+        for (let i = 0; i < limit; i++) {
+          tags.push({ text: `tag${i}` });
+        }
+        return tags;
+      });
+    });
+
+    it('uses limited metrics and tags list', async () => {
+      ctx.state.supportsTags = true;
+      const segments = await getAltSegmentsSelectables(ctx.state, 0, '');
+      expect(segments).toHaveLength(10000);
+      expect(segments[0].value!.value).toBe('*'); // * - is a fixed metric name, always added at the top
+      expect(segments[4999].value!.value).toBe('metric4998');
+      expect(segments[5000].value!.value).toBe('tag: tag0');
+      expect(segments[9999].value!.value).toBe('tag: tag4999');
+    });
+
+    it('uses correct limit for metrics and tags list when tags are not supported', async () => {
+      ctx.state.supportsTags = false;
+      const segments = await getAltSegmentsSelectables(ctx.state, 0, '');
+      expect(segments).toHaveLength(5000);
+      expect(segments[0].value!.value).toBe('*'); // * - is a fixed metric name, always added at the top
+      expect(segments[4999].value!.value).toBe('metric4998');
+    });
+
+    it('uses limited metrics when adding more metrics', async () => {
+      const segments = await getAltSegmentsSelectables(ctx.state, 1, '');
+      expect(segments).toHaveLength(5000);
+    });
+  });
 });

public/app/plugins/datasource/graphite/state/providers.ts

@@ -10,6 +10,15 @@ import { GraphiteSegment, GraphiteTag, GraphiteTagOperator } from '../types';
 import { mapSegmentsToSelectables, mapStringsToSelectables } from '../components/helpers';
 import { SelectableValue } from '@grafana/data';
 
+/**
+ * All auto-complete lists are updated while typing. To avoid performance issues we do not render more
+ * than MAX_SUGGESTIONS limits in a single dropdown.
+ *
+ * MAX_SUGGESTIONS is per metrics and tags separately. On the very first dropdown where metrics and tags are
+ * combined together meaning it may end up with max of 2 * MAX_SUGGESTIONS items in total.
+ */
+const MAX_SUGGESTIONS = 5000;
+
 /**
  * Providers are hooks for views to provide temporal data for autocomplete. They don't modify the state.
  */
@@ -72,8 +81,10 @@ async function getAltSegments(
       });
     });
 
-    // add wildcard option
+    // add wildcard option and limit number of suggestions (API doesn't support limiting
+    // hence we are doing it here)
     altSegments.unshift({ value: '*', expandable: true });
+    altSegments.splice(MAX_SUGGESTIONS);
 
     if (state.supportsTags && index === 0) {
       removeTaggedEntry(altSegments);
@@ -88,6 +99,10 @@ async function getAltSegments(
   return [];
 }
 
+/**
+ * Get the list of segments with tags and metrics. Suggestions are reduced in getAltSegments and addAltTagSegments so in case
+ * we hit MAX_SUGGESTIONS limit there are always some tags and metrics shown.
+ */
 export async function getAltSegmentsSelectables(
   state: GraphiteQueryEditorState,
   index: number,
@@ -106,7 +121,10 @@ export function getTagOperatorsSelectables(): Array<SelectableValue<GraphiteTagO
 async function getTags(state: GraphiteQueryEditorState, index: number, tagPrefix: string): Promise<string[]> {
   try {
     const tagExpressions = state.queryModel.renderTagExpressions(index);
-    const values = await state.datasource.getTagsAutoComplete(tagExpressions, tagPrefix, { range: state.range });
+    const values = await state.datasource.getTagsAutoComplete(tagExpressions, tagPrefix, {
+      range: state.range,
+      limit: MAX_SUGGESTIONS,
+    });
 
     const altTags = map(values, 'text');
     altTags.splice(0, 0, state.removeTagValue);
@@ -134,7 +152,10 @@ async function getTagsAsSegments(state: GraphiteQueryEditorState, tagPrefix: str
   let tagsAsSegments: GraphiteSegment[];
   try {
     const tagExpressions = state.queryModel.renderTagExpressions();
-    const values = await state.datasource.getTagsAutoComplete(tagExpressions, tagPrefix, { range: state.range });
+    const values = await state.datasource.getTagsAutoComplete(tagExpressions, tagPrefix, {
+      range: state.range,
+      limit: MAX_SUGGESTIONS,
+    });
     tagsAsSegments = map(values, (val) => {
       return {
         value: val.text,
@@ -150,6 +171,9 @@ async function getTagsAsSegments(state: GraphiteQueryEditorState, tagPrefix: str
   return tagsAsSegments;
 }
 
+/**
+ * Get list of tags, used when adding additional tags (first tag is selected from a joined list of metrics and tags)
+ */
 export async function getTagsAsSegmentsSelectables(
   state: GraphiteQueryEditorState,
   tagPrefix: string
@@ -165,7 +189,9 @@ async function getTagValues(
 ): Promise<string[]> {
   const tagExpressions = state.queryModel.renderTagExpressions(index);
   const tagKey = tag.key;
-  const values = await state.datasource.getTagValuesAutoComplete(tagExpressions, tagKey, valuePrefix, {});
+  const values = await state.datasource.getTagValuesAutoComplete(tagExpressions, tagKey, valuePrefix, {
+    limit: MAX_SUGGESTIONS,
+  });
   const altValues = map(values, 'text');
   // Add template variables as additional values
   eachRight(state.templateSrv.getVariables(), (variable) => {

public/test/specs/helpers.ts

@@ -150,6 +150,9 @@ export class ContextSrvStub {
 
 export function TemplateSrvStub(this: any) {
   this.variables = [];
+  this.getVariables = function () {
+    return this.variables;
+  };
   this.templateSettings = { interpolate: /\[\[([\s\S]+?)\]\]/g };
   this.data = {};
   this.replace = (text: string) => {