InfluxDB: Add an optional time range filter for tag queries in the query panel autocompleteInflux tag filter (#107195)

* InfluxDB: Add an optional time range filter for tag queries in the query panel autocomplete

* Betterer updates

* Update test

---------

Co-authored-by: Nikolay Tsvetkov <nikolay.cvetkov@gmail.com>

.betterer.results

@@ -3606,13 +3606,6 @@ exports[`better eslint`] = {
       [0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "0"],
       [0, 0, 0, "Do not use any type assertions.", "1"]
     ],
-    "public/app/plugins/datasource/influxdb/components/editor/config/InfluxInfluxQLConfig.tsx:5381": [
-      [0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "0"],
-      [0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "1"],
-      [0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "2"],
-      [0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "3"],
-      [0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "4"]
-    ],
     "public/app/plugins/datasource/influxdb/components/editor/config/InfluxSQLConfig.tsx:5381": [
       [0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "0"],
       [0, 0, 0, "Add noMargin prop to Field components to remove built-in margins. Use layout components like Stack or Grid with the gap prop instead for consistent spacing.", "1"],

docs/sources/datasources/influxdb/configure-influxdb-data-source/_index.md

@@ -126,6 +126,7 @@ The following settings are specific to the InfluxQL query language option.
 - **Password** - Defines the token used to query the bucket defined in **Database**. Retrieve the password from the [Tokens page](https://docs.influxdata.com/influxdb/v2.0/security/tokens/view-tokens/) of the InfluxDB UI.
 - **HTTP method** - Sets the HTTP method used to query your data source. The POST method allows for larger queries that would return an error using the GET method. The default method is `POST`.
 - **Min time interval** - _(Optional)_ Sets the minimum time interval for auto group-by. Grafana recommends setting this to match the data write frequency. For example, if your data is written every minute, it’s recommended to set this interval to 1 minute, so that each group contains data from each new write. The default is `10s`. Refer to [Min time interval](#min-time-interval) for format examples.
+- **Autocomplete range** - _(Optional)_ Sets a time range limit for the query editor's autocomplete to reduce the execution time of tag filter queries. As a result, any tags not present within the defined time range will be filtered out. For example, setting the value to 12h will include only tag keys/values from the past 12 hours. This feature is recommended for use with very large databases, where significant performance improvements can be observed.
 - **Max series** - _(Optional)_ Sets a limit on the maximum number of series or tables that Grafana processes. Set a lower limit to prevent system overload, or increase it if you have many small time series and need to display more of them. The default is `1000`.
 
 ### SQL-specific configuration section

public/app/plugins/datasource/influxdb/components/editor/config/ConfigEditor.test.tsx

@@ -36,6 +36,7 @@ const setup = (optionOverrides?: object) => {
       jsonData: {
         httpMode: 'POST',
         timeInterval: '4',
+        showTagTime: '3h',
       },
       secureJsonFields: {},
       version: 1,

public/app/plugins/datasource/influxdb/components/editor/config/InfluxInfluxQLConfig.tsx

@@ -50,6 +50,7 @@ export const InfluxInfluxQLConfig = (props: Props) => {
         label={<InlineLabel width={WIDTH_SHORT}>Database</InlineLabel>}
         className={styles.horizontalField}
         htmlFor={`${htmlPrefix}-db`}
+        noMargin
       >
         <Input
           id={`${htmlPrefix}-db`}
@@ -72,6 +73,7 @@ export const InfluxInfluxQLConfig = (props: Props) => {
         label={<InlineLabel width={WIDTH_SHORT}>User</InlineLabel>}
         className={styles.horizontalField}
         htmlFor={`${htmlPrefix}-user`}
+        noMargin
       >
         <Input
           id={`${htmlPrefix}-user`}
@@ -84,6 +86,7 @@ export const InfluxInfluxQLConfig = (props: Props) => {
         horizontal
         label={<InlineLabel width={WIDTH_SHORT}>Password</InlineLabel>}
         className={styles.horizontalField}
+        noMargin
       >
         <SecretInput
           isConfigured={Boolean(secureJsonFields && secureJsonFields.password)}
@@ -109,6 +112,7 @@ export const InfluxInfluxQLConfig = (props: Props) => {
         }
         htmlFor={`${htmlPrefix}-http-method`}
         className={styles.horizontalField}
+        noMargin
       >
         <Select
           inputId={`${htmlPrefix}-http-method`}
@@ -131,6 +135,7 @@ export const InfluxInfluxQLConfig = (props: Props) => {
           </InlineLabel>
         }
         className={styles.horizontalField}
+        noMargin
       >
         <Input
           className="width-20"
@@ -139,6 +144,27 @@ export const InfluxInfluxQLConfig = (props: Props) => {
           onChange={onUpdateDatasourceJsonDataOption(props, 'timeInterval')}
         />
       </Field>
+
+      <Field
+        horizontal
+        label={
+          <InlineLabel
+            width={WIDTH_SHORT}
+            tooltip="This time range is used in the query editor's autocomplete to reduce the execution time of tag filter queries."
+          >
+            Autocomplete range
+          </InlineLabel>
+        }
+        className={styles.horizontalField}
+        noMargin
+      >
+        <Input
+          className="width-20"
+          placeholder="12h"
+          value={options.jsonData.showTagTime || ''}
+          onChange={onUpdateDatasourceJsonDataOption(props, 'showTagTime')}
+        />
+      </Field>
     </>
   );
 };

public/app/plugins/datasource/influxdb/datasource.ts

@@ -64,6 +64,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
   httpMode: string;
   version?: InfluxVersion;
   isProxyAccess: boolean;
+  showTagTime: string;
 
   constructor(
     instanceSettings: DataSourceInstanceSettings<InfluxOptions>,
@@ -85,6 +86,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
     const settingsData: InfluxOptions = instanceSettings.jsonData ?? {};
     this.database = settingsData.dbName ?? instanceSettings.database;
     this.interval = settingsData.timeInterval;
+    this.showTagTime = settingsData.showTagTime || '';
     this.httpMode = settingsData.httpMode || 'GET';
     this.responseParser = new ResponseParser();
     this.version = settingsData.version ?? InfluxVersion.InfluxQL;
@@ -441,6 +443,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
       type: 'TAG_KEYS',
       templateService: this.templateSrv,
       database: this.database,
+      withTimeFilter: this.showTagTime,
     });
 
     return this.metricFindQuery({ refId: 'get-tag-keys', query });
@@ -452,6 +455,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
       templateService: this.templateSrv,
       database: this.database,
       withKey: options.key,
+      withTimeFilter: this.showTagTime,
     });
 
     return this.metricFindQuery({ refId: 'get-tag-values', query });

public/app/plugins/datasource/influxdb/influxql_metadata_query.ts

@@ -14,6 +14,7 @@ type MetadataQueryOptions = {
   tags?: InfluxQueryTag[];
   withKey?: string;
   withMeasurementFilter?: string;
+  withTimeFilter?: string;
 };
 
 const runExploreQuery = async (options: MetadataQueryOptions): Promise<Array<{ text: string }>> => {
@@ -26,6 +27,7 @@ const runExploreQuery = async (options: MetadataQueryOptions): Promise<Array<{ t
     tags,
     withKey,
     withMeasurementFilter,
+    withTimeFilter: datasource.showTagTime,
     templateService: datasource.templateSrv,
     database: datasource.database,
   });

public/app/plugins/datasource/influxdb/influxql_query_builder.test.ts

@@ -119,6 +119,28 @@ describe('influxql-query-builder', () => {
       });
       expect(query).toBe(`SHOW TAG KEYS WHERE "app" == ''`);
     });
+
+    it('should have where condition in tag keys query with time filter', () => {
+      const query = buildMetadataQuery({
+        type: 'TAG_KEYS',
+        templateService,
+        measurement: '',
+        tags: [],
+        withTimeFilter: '3h',
+      });
+      expect(query).toBe('SHOW TAG KEYS WHERE time > now() - 3h');
+    });
+
+    it('should have and in  where condition in tag keys query with tags and time filter', () => {
+      const query = buildMetadataQuery({
+        type: 'TAG_KEYS',
+        templateService,
+        measurement: '',
+        tags: [{ key: 'host', value: 'se1' }],
+        withTimeFilter: '3h',
+      });
+      expect(query).toBe('SHOW TAG KEYS WHERE "host" = \'se1\' AND time > now() - 3h');
+    });
   });
 
   describe('TAG_VALUES', () => {
@@ -184,6 +206,37 @@ describe('influxql-query-builder', () => {
       });
       expect(query).toBe('SHOW TAG VALUES FROM "cpu" WITH KEY = "app" WHERE "host" =~ /server.*/');
     });
+
+    it('should add time filter with and if with time filter', () => {
+      const query = buildMetadataQuery({
+        type: 'TAG_VALUES',
+        templateService,
+        withKey: 'app',
+        measurement: 'cpu',
+        retentionPolicy: 'one_week',
+        tags: [],
+        withTimeFilter: '2h',
+      });
+      expect(query).toBe('SHOW TAG VALUES FROM "one_week"."cpu" WITH KEY = "app" WHERE time > now() - 2h');
+    });
+
+    it('should add time filter if with time filter and tags', () => {
+      const query = buildMetadataQuery({
+        type: 'TAG_VALUES',
+        templateService,
+        withKey: 'app',
+        measurement: 'cpu',
+        retentionPolicy: 'one_week',
+        tags: [
+          { key: 'app', value: 'email' },
+          { key: 'host', value: 'server1' },
+        ],
+        withTimeFilter: '2h',
+      });
+      expect(query).toBe(
+        'SHOW TAG VALUES FROM "one_week"."cpu" WITH KEY = "app" WHERE "host" = \'server1\' AND time > now() - 2h'
+      );
+    });
   });
 
   describe('MEASUREMENTS', () => {
@@ -271,6 +324,18 @@ describe('influxql-query-builder', () => {
       });
       expect(query).toBe(`SHOW MEASUREMENTS WHERE "app" == '' LIMIT 100`);
     });
+
+    it('should not add time filter when getting measurements', () => {
+      const query = buildMetadataQuery({
+        type: 'MEASUREMENTS',
+        templateService,
+        database: undefined,
+        measurement: undefined,
+        tags: [{ key: 'app', value: '', operator: '==' }],
+        withTimeFilter: '2h',
+      });
+      expect(query).toBe(`SHOW MEASUREMENTS WHERE "app" == '' LIMIT 100`);
+    });
   });
 
   it('should not add FROM statement if the measurement empty', () => {

public/app/plugins/datasource/influxdb/influxql_query_builder.ts

@@ -15,6 +15,7 @@ export const buildMetadataQuery = (params: {
   tags?: InfluxQueryTag[];
   withKey?: string;
   withMeasurementFilter?: string;
+  withTimeFilter?: string;
 }): string => {
   let query = '';
   let {
@@ -27,6 +28,7 @@ export const buildMetadataQuery = (params: {
     tags,
     withKey,
     withMeasurementFilter,
+    withTimeFilter,
   } = params;
 
   switch (type) {
@@ -89,8 +91,9 @@ export const buildMetadataQuery = (params: {
     query += ' WITH KEY = "' + keyIdentifier + '"';
   }
 
+  let whereConditions: string[] = [];
   if (tags && tags.length > 0) {
-    const whereConditions = reduce<InfluxQueryTag, string[]>(
+    whereConditions = reduce<InfluxQueryTag, string[]>(
       tags,
       (memo, tag) => {
         // do not add a condition for the key we want to explore for
@@ -108,10 +111,13 @@ export const buildMetadataQuery = (params: {
       },
       []
     );
+  }
 
-    if (whereConditions.length > 0) {
-      query += ' WHERE ' + whereConditions.join(' ');
-    }
+  let shouldUseTime = isValidTimeFilter(withTimeFilter) && type !== 'MEASUREMENTS';
+  if (whereConditions.length > 0) {
+    query += ' WHERE ' + whereConditions.join(' ') + (shouldUseTime ? ' AND time > now() - ' + withTimeFilter : '');
+  } else {
+    query += shouldUseTime ? ' WHERE time > now() - ' + withTimeFilter : '';
   }
 
   if (type === 'MEASUREMENTS') {
@@ -124,6 +130,34 @@ export const buildMetadataQuery = (params: {
   return query;
 };
 
+// Function to validate the provided time filter value
+export function isValidTimeFilter(value?: string): boolean {
+  if (!value || typeof value !== 'string') {
+    return false;
+  }
+
+  // Normalize microseconds unit
+  const normalizedValue = value.replace(/µ/g, 'u');
+
+  const validUnits = new Set(['ns', 'u', 'ms', 's', 'm', 'h', 'd', 'w']);
+  const pattern = /(\d+)(ns|u|ms|s|m|h|d|w)/g;
+  const usedUnits = new Set<string>();
+
+  let match: RegExpExecArray | null;
+  let totalLength = 0;
+
+  while ((match = pattern.exec(normalizedValue)) !== null) {
+    const [fullMatch, numberPart, unit] = match;
+    totalLength += fullMatch.length;
+
+    if (!validUnits.has(unit) || usedUnits.has(unit) || numberPart.startsWith('0')) {
+      return false;
+    }
+    usedUnits.add(unit);
+  }
+  return totalLength === normalizedValue.length;
+}
+
 // A merge of query_builder/renderTagCondition and influx_query_model/renderTagCondition
 export function renderTagCondition(
   tag: InfluxQueryTag,

public/app/plugins/datasource/influxdb/types.ts

@@ -13,6 +13,7 @@ export interface InfluxOptions extends DataSourceJsonData {
 
   timeInterval?: string;
   httpMode?: string;
+  showTagTime?: string;
 
   dbName?: string;
   product?: string;