Search: use minisearch for better frontend searching (#46710)

Co-authored-by: nmarrs <nathanielmarrs@gmail.com>
2022-03-17 14:26:48 -07:00
parent 75b3c96bef
commit 736820f975
11 changed files with 7711 additions and 117 deletions
@@ -315,6 +315,7 @@
    "logfmt": "^1.3.2",
    "lru-cache": "7.5.1",
    "memoize-one": "6.0.0",
+    "minisearch": "5.0.0-beta1",
    "moment": "2.29.1",
    "moment-timezone": "0.5.34",
    "monaco-editor": "^0.31.1",
@@ -8,11 +8,8 @@ import { css } from '@emotion/css';

 import Page from 'app/core/components/Page/Page';
 import { SearchPageDashboards } from './SearchPageDashboards';
-import { SearchPageDashboardList } from './SearchPageDashboardList';
 import { loadResults } from './state/actions';
 import { StoreState } from 'app/types';
-import { SearchPageStats } from './SearchPageStats';
-import { buildStatsTable } from './data';

 const node: NavModelItem = {
  id: 'search',
@@ -25,8 +22,7 @@ export default function SearchPage() {
  const dispatch = useDispatch();
  const styles = useStyles2(getStyles);

-  const dashboards = useSelector((state: StoreState) => state.searchPage.data.dashboards);
-  const panels = useSelector((state: StoreState) => state.searchPage.data.panels);
+  const results = useSelector((state: StoreState) => state.searchPage.data.results);

  const [query, setQuery] = useState('');

@@ -47,25 +43,19 @@ export default function SearchPage() {
      <Page.Contents>
        <Input value={query} onChange={(e) => setQuery(e.currentTarget.value)} autoFocus spellCheck={false} />
        <br /> <br />
-        {!dashboards && <div>Loading....</div>}
-        {dashboards && (
+        {!results && query && <div>Loading....</div>}
+        <div>
+          FIELDS: {results?.body.fields.length}
+          <br />
+          RESULT: {results?.body.length}
+        </div>
+        {results?.body && (
          <div>
            <AutoSizer style={{ width: '100%', height: '1000px' }}>
              {({ width }) => {
                return (
                  <div>
-                    {dashboards && <SearchPageDashboardList dashboards={dashboards} />}
-                    <br />
-                    {dashboards.dataFrame && dashboards.dataFrame.length > 0 && (
-                      <SearchPageDashboards dashboards={dashboards.dataFrame} width={width} />
-                    )}
-
-                    {panels && (
-                      <SearchPageStats
-                        panelTypes={buildStatsTable(panels.fields.find((f) => f.name === 'Type'))}
-                        width={width}
-                      />
-                    )}
+                    <SearchPageDashboards dashboards={results?.body!} width={width} />
                  </div>
                );
              }}
@@ -1,29 +0,0 @@
-import React from 'react';
-import { DataFrameView, GrafanaTheme2 } from '@grafana/data';
-import { css } from '@emotion/css';
-import { useStyles2 } from '@grafana/ui';
-
-type Props = {
-  dashboards: DataFrameView;
-};
-
-export const SearchPageDashboardList = ({ dashboards }: Props) => {
-  const styles = useStyles2(getStyles);
-
-  return (
-    <div className={styles.listContainer}>
-      {dashboards.map((dash) => (
-        <div key={dash.UID}>
-          <a href={dash.URL}>{dash.Name}</a>
-        </div>
-      ))}
-    </div>
-  );
-};
-
-const getStyles = (theme: GrafanaTheme2) => ({
-  listContainer: css`
-    max-height: 300px;
-    overflow: scroll;
-  `,
-});
@@ -1,42 +0,0 @@
-import React from 'react';
-import { DataFrame, GrafanaTheme2, LoadingState } from '@grafana/data';
-import { PanelRenderer } from '@grafana/runtime';
-import { css } from '@emotion/css';
-import { useStyles2 } from '@grafana/ui';
-
-type Props = {
-  panelTypes: DataFrame;
-  width: number;
-};
-
-export const SearchPageStats = ({ panelTypes, width }: Props) => {
-  const styles = useStyles2(getStyles);
-
-  return (
-    <>
-      <h1>Stats</h1>
-      <table className={styles.table}>
-        <tr>
-          <td>
-            <PanelRenderer
-              pluginId="table"
-              title="Panels"
-              data={{ series: [panelTypes], state: LoadingState.Done } as any}
-              options={{}}
-              width={width / 2}
-              height={200}
-              fieldConfig={{ defaults: {}, overrides: [] }}
-              timeZone="browser"
-            />
-          </td>
-        </tr>
-      </table>
-    </>
-  );
-};
-
-const getStyles = (theme: GrafanaTheme2) => ({
-  table: css`
-    width: 100%;
-  `,
-});
@@ -1,37 +1,27 @@
-import { DataFrameView } from '@grafana/data';
-
+import { reduceField } from '@grafana/data';
 import { ThunkResult } from 'app/types';
-import { getDashboardData, filterDataFrame } from '../data';
-import { DashboardResult } from '../types';
+import { getRawIndexData, getFrontendGrafanaSearcher } from '../../service/frontend';
 import { fetchResults } from './reducers';

 export const loadResults = (query: string): ThunkResult<void> => {
  return async (dispatch) => {
-    const data = await getDashboardData();
-
-    if (!data.dashboards || !data.panels) {
+    const data = await getRawIndexData();
+    if (!data.dashboard) {
      return;
    }

-    if (!data.dashboards.length || !query.length) {
-      return dispatch(
-        fetchResults({
-          data: {
-            dashboards: new DataFrameView<DashboardResult>(data.dashboards),
-            panels: data.panels,
-          },
-        })
-      );
-    }
+    const searcher = getFrontendGrafanaSearcher(data);
+    const results = await searcher.search(query);

-    const dashboards = filterDataFrame(query, data.dashboards, 'Name', 'Description', 'Tags');
-    const panels = filterDataFrame(query, data.panels, 'Name', 'Description', 'Type');
+    // HACK avoid redux error!
+    results.body.fields.forEach((f) => {
+      reduceField({ field: f, reducers: ['min', 'max'] });
+    });

    return dispatch(
      fetchResults({
        data: {
-          dashboards: new DataFrameView<DashboardResult>(dashboards),
-          panels: panels,
+          results,
        },
      })
    );
@@ -1,19 +1,16 @@
 import { createSlice, PayloadAction } from '@reduxjs/toolkit';
-import { DataFrame, DataFrameView } from '@grafana/data';

-import { DashboardResult } from '../types';
+import { QueryResponse } from '../../service/types';

 export interface SearchPageState {
  data: {
-    dashboards: DataFrameView<DashboardResult> | null;
-    panels: DataFrame | null;
+    results?: QueryResponse;
  };
 }

 export const initialState: SearchPageState = {
  data: {
-    dashboards: null,
-    panels: null,
+    results: undefined,
  },
 };

@@ -0,0 +1,72 @@
+import { dataFrameFromJSON, DataFrameJSON, toDataFrame } from '@grafana/data';
+
+import { dump } from './testData';
+import { getFrontendGrafanaSearcher, RawIndexData } from './frontend';
+
+describe('simple search', () => {
+  it('should support frontend search', async () => {
+    const raw: RawIndexData = {
+      dashboard: toDataFrame([
+        { Name: 'A name (dash)', Description: 'A descr (dash)' },
+        { Name: 'B name (dash)', Description: 'B descr (dash)' },
+      ]),
+      panel: toDataFrame([
+        { Name: 'A name (panels)', Description: 'A descr (panels)' },
+        { Name: 'B name (panels)', Description: 'B descr (panels)' },
+      ]),
+    };
+
+    const searcher = getFrontendGrafanaSearcher(raw);
+    let results = await searcher.search('name');
+    expect(results.body.fields[1].values.toArray()).toMatchInlineSnapshot(`
+      Array [
+        "A name (dash)",
+        "B name (dash)",
+        "A name (panels)",
+        "B name (panels)",
+      ]
+    `);
+
+    results = await searcher.search('B');
+    expect(results.body.fields[1].values.toArray()).toMatchInlineSnapshot(`
+      Array [
+        "B name (dash)",
+        "B name (panels)",
+      ]
+    `);
+  });
+
+  it('should work with query results', async () => {
+    const data: RawIndexData = {};
+    for (const frame of dump.results.A.frames) {
+      switch (frame.schema.name) {
+        case 'folder':
+        case 'folders':
+          data.folder = dataFrameFromJSON(frame as DataFrameJSON);
+          break;
+
+        case 'dashboards':
+        case 'dashboard':
+          data.dashboard = dataFrameFromJSON(frame as DataFrameJSON);
+          break;
+
+        case 'panels':
+        case 'panel':
+          data.panel = dataFrameFromJSON(frame as DataFrameJSON);
+          break;
+      }
+    }
+
+    const searcher = getFrontendGrafanaSearcher(data);
+
+    const results = await searcher.search('automation');
+    expect(results.body.fields[1].values.toArray()).toMatchInlineSnapshot(`
+      Array [
+        "Home automation",
+        "Panel name with automation",
+        "Tides",
+        "Gaps & null between every point for series B",
+      ]
+    `);
+  });
+});
@@ -0,0 +1,249 @@
+import MiniSearch, { SearchResult } from 'minisearch';
+import { lastValueFrom } from 'rxjs';
+import { ArrayVector, DataFrame, FieldType, Vector } from '@grafana/data';
+import { getDataSourceSrv } from '@grafana/runtime';
+
+import { GrafanaDatasource } from 'app/plugins/datasource/grafana/datasource';
+import { GrafanaQueryType } from 'app/plugins/datasource/grafana/types';
+import { GrafanaSearcher, QueryFilters, QueryResponse } from './types';
+
+export interface RawIndexData {
+  dashboard?: DataFrame;
+  panel?: DataFrame;
+  folder?: DataFrame;
+}
+
+export type SearchResultKind = keyof RawIndexData;
+
+interface InputDoc {
+  kind: SearchResultKind;
+  index: number;
+
+  // Fields
+  id?: Vector<number>;
+  url?: Vector<string>;
+  uid?: Vector<string>;
+  name?: Vector<string>;
+  description?: Vector<string>;
+  dashboardID?: Vector<number>;
+  type?: Vector<string>;
+  tags?: Vector<string>; // JSON strings?
+}
+
+interface CompositeKey {
+  kind: SearchResultKind;
+  index: number;
+}
+
+type Lookup = Map<SearchResultKind, InputDoc>;
+
+const generateResults = (rawResults: SearchResult[], lookup: Lookup): DataFrame => {
+  // frame fields
+  const url: string[] = [];
+  const kind: string[] = [];
+  const type: string[] = [];
+  const name: string[] = [];
+  const info: any[] = [];
+  const score: number[] = [];
+
+  for (const res of rawResults) {
+    const key = res.id as CompositeKey;
+    const index = key.index;
+    const input = lookup.get(key.kind);
+    if (!input) {
+      continue;
+    }
+
+    url.push(input.url?.get(index) ?? '?');
+    kind.push(key.kind);
+    name.push(input.name?.get(index) ?? '?');
+    type.push(input.type?.get(index) as any);
+    info.push(res.match); // ???
+    score.push(res.score);
+  }
+
+  return {
+    fields: [
+      { name: 'Kind', config: {}, type: FieldType.string, values: new ArrayVector(kind) },
+      { name: 'Name', config: {}, type: FieldType.string, values: new ArrayVector(name) },
+      {
+        name: 'URL',
+        config: {
+          links: [
+            {
+              title: 'view',
+              url: '?',
+              onClick: (evt) => {
+                const { field, rowIndex } = evt.origin;
+                if (field && rowIndex != null) {
+                  const url = field.values.get(rowIndex) as string;
+                  window.location.href = url; // HACK!
+                }
+              },
+            },
+          ],
+        },
+        type: FieldType.string,
+        values: new ArrayVector(url),
+      },
+      { name: 'type', config: {}, type: FieldType.other, values: new ArrayVector(type) },
+      { name: 'info', config: {}, type: FieldType.other, values: new ArrayVector(info) },
+      { name: 'score', config: {}, type: FieldType.number, values: new ArrayVector(score) },
+    ],
+    length: url.length,
+  };
+};
+
+export function getFrontendGrafanaSearcher(data: RawIndexData): GrafanaSearcher {
+  const searcher = new MiniSearch<InputDoc>({
+    idField: '__id',
+    fields: ['name', 'description', 'tags'], // fields to index for full-text search
+    searchOptions: {
+      boost: {
+        name: 3,
+        description: 1,
+      },
+      // boost dashboard matches first
+      boostDocument: (documentId: any, term: string) => {
+        const kind = documentId.kind;
+        if (kind === 'dashboard') {
+          return 1.4;
+        }
+        if (kind === 'folder') {
+          return 1.2;
+        }
+        return 1;
+      },
+      prefix: true, // (term) => term.length > 3,
+      fuzzy: (term) => (term.length > 4 ? 0.2 : false),
+    },
+    extractField: (doc, name) => {
+      // return a composite key for the id
+      if (name === '__id') {
+        return {
+          kind: doc.kind,
+          index: doc.index,
+        };
+      }
+      const values = (doc as any)[name] as Vector;
+      if (!values) {
+        return undefined;
+      }
+      return values.get(doc.index);
+    },
+  });
+
+  const lookup: Lookup = new Map<SearchResultKind, InputDoc>();
+  for (const [key, frame] of Object.entries(data)) {
+    const kind = key as SearchResultKind;
+    const input = getInputDoc(kind, frame);
+    lookup.set(kind, input);
+    for (let i = 0; i < frame.length; i++) {
+      input.index = i;
+      searcher.add(input);
+    }
+  }
+
+  // Construct the URL field for each panel
+  if (true) {
+    const dashboard = lookup.get('dashboard');
+    const panel = lookup.get('panel');
+    if (dashboard?.id && panel?.dashboardID && dashboard.url) {
+      const dashIDToIndex = new Map<number, number>();
+      for (let i = 0; i < dashboard.id?.length; i++) {
+        dashIDToIndex.set(dashboard.id.get(i), i);
+      }
+
+      const urls: string[] = new Array(panel.dashboardID.length);
+      for (let i = 0; i < panel.dashboardID.length; i++) {
+        const dashboardID = panel.dashboardID.get(i);
+        const index = dashIDToIndex.get(dashboardID);
+        if (index != null) {
+          urls[i] = dashboard.url.get(index) + '?viewPanel=' + panel.id?.get(i);
+        }
+      }
+      panel.url = new ArrayVector(urls);
+    }
+  }
+
+  return {
+    search: async (query: string, filter?: QueryFilters) => {
+      const found = searcher.search(query);
+
+      const results = generateResults(found, lookup);
+
+      const searchResult: QueryResponse = {
+        body: results,
+      };
+
+      return searchResult;
+    },
+  };
+}
+
+export function getInputDoc(kind: SearchResultKind, frame: DataFrame): InputDoc {
+  const input: InputDoc = {
+    kind,
+    index: 0,
+  };
+  for (const field of frame.fields) {
+    switch (field.name) {
+      case 'name':
+      case 'Name':
+        input.name = field.values;
+        break;
+      case 'Description':
+      case 'Description':
+        input.description = field.values;
+        break;
+      case 'url':
+      case 'URL':
+        input.url = field.values;
+        break;
+      case 'uid':
+      case 'UID':
+        input.uid = field.values;
+        break;
+      case 'id':
+      case 'ID':
+        input.id = field.values;
+        break;
+      case 'DashboardID':
+      case 'dashboardID':
+        input.dashboardID = field.values;
+        break;
+      case 'Type':
+      case 'type':
+        input.type = field.values;
+        break;
+    }
+  }
+  return input;
+}
+
+export async function getRawIndexData(): Promise<RawIndexData> {
+  const ds = (await getDataSourceSrv().get('-- Grafana --')) as GrafanaDatasource;
+  const rsp = await lastValueFrom(
+    ds.query({
+      targets: [
+        { refId: 'A', queryType: GrafanaQueryType.Search }, // gets all data
+      ],
+    } as any)
+  );
+
+  const data: RawIndexData = {};
+  for (const f of rsp.data) {
+    switch (f.name) {
+      case 'dashboards':
+        data.dashboard = f;
+        break;
+      case 'panels':
+        data.panel = f;
+        break;
+      case 'folders':
+        data.folder = f;
+        break;
+    }
+  }
+  return data;
+}
@@ -0,0 +1,24 @@
+import { DataFrame } from '@grafana/data';
+
+export interface QueryResult {
+  kind: string; // panel, dashboard, folder
+  name: string;
+  description?: string;
+  url: string; // link to value (unique)
+  tags?: string[];
+  location?: string; // the folder name
+  score?: number;
+}
+
+export interface QueryFilters {
+  kind?: string; // limit to a single type
+  tags?: string[]; // match all tags
+}
+
+export interface QueryResponse {
+  body: DataFrame;
+}
+
+export interface GrafanaSearcher {
+  search: (query: string, filter?: QueryFilters) => Promise<QueryResponse>;
+}
@@ -20947,6 +20947,7 @@ __metadata:
    lru-cache: 7.5.1
    memoize-one: 6.0.0
    mini-css-extract-plugin: 2.6.0
+    minisearch: 5.0.0-beta1
    moment: 2.29.1
    moment-timezone: 0.5.34
    monaco-editor: ^0.31.1
@@ -26360,6 +26361,13 @@ __metadata:
  languageName: node
  linkType: hard

+"minisearch@npm:5.0.0-beta1":
+  version: 5.0.0-beta1
+  resolution: "minisearch@npm:5.0.0-beta1"
+  checksum: 7c5ba8b2d1b52df0724e69183306b4204ae4cdc0102813da7f12a8f90a7b4efe7add4c54814ec1bb63f8bd5be599373af9055e0d68ceeef3d225f0a2c7637699
+  languageName: node
+  linkType: hard
+
 "minizlib@npm:^1.3.3":
  version: 1.3.3
  resolution: "minizlib@npm:1.3.3"