From afa40190d4f446a3842fdf4ee263d5f7e77fa4a5 Mon Sep 17 00:00:00 2001 From: "Grot (@grafanabot)" <43478413+grafanabot@users.noreply.github.com> Date: Thu, 26 May 2022 20:30:58 -0400 Subject: [PATCH] SearchV2: improve searcher API, and include a fallback SQL based implementation (#49535) (#49746) (cherry picked from commit a641949a05448869922cdbc15a6a55da971a785f) Co-authored-by: Ryan McKinley --- pkg/services/searchV2/bluge.go | 21 -- .../search/components/DashboardListPage.tsx | 3 +- .../search/components/DashboardSearch.tsx | 2 +- .../search/page/components/ActionRow.tsx | 4 +- .../search/page/components/FolderSection.tsx | 15 +- .../search/page/components/FolderView.tsx | 19 +- .../search/page/components/SearchView.tsx | 1 + .../search/page/components/columns.tsx | 7 +- .../search/page/components/sorting.ts | 37 ---- public/app/features/search/service/bluge.ts | 56 ++++- .../app/features/search/service/searcher.ts | 8 +- public/app/features/search/service/sql.ts | 198 ++++++++++++++++++ public/app/features/search/service/types.ts | 6 +- 13 files changed, 282 insertions(+), 95 deletions(-) delete mode 100644 public/app/features/search/page/components/sorting.ts create mode 100644 public/app/features/search/service/sql.ts diff --git a/pkg/services/searchV2/bluge.go b/pkg/services/searchV2/bluge.go index d4c16ea1761..e3e542efc83 100644 --- a/pkg/services/searchV2/bluge.go +++ b/pkg/services/searchV2/bluge.go @@ -295,27 +295,6 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader, response := &backend.DataResponse{} header := &customMeta{} - // Folder listing structure. - idx := strings.Index(q.Query, ":") - if idx > 0 { - key := q.Query[0:idx] - val := q.Query[idx+1:] - if key == "list" { - q.Limit = 1000 - q.Query = "" - q.Location = "" - q.Explain = false - q.SkipLocation = true - q.Facet = nil - if val == "root" || val == "" { - q.Kind = []string{string(entityKindFolder)} - } else { - q.Location = val - q.Kind = []string{string(entityKindDashboard)} - } - } - } - hasConstraints := false fullQuery := bluge.NewBooleanQuery() fullQuery.AddMust(newPermissionFilter(filter, logger)) diff --git a/public/app/features/search/components/DashboardListPage.tsx b/public/app/features/search/components/DashboardListPage.tsx index ffb8219067e..3ececc199ca 100644 --- a/public/app/features/search/components/DashboardListPage.tsx +++ b/public/app/features/search/components/DashboardListPage.tsx @@ -46,8 +46,7 @@ export const DashboardListPage: FC = memo(({ navModel, match, location }) return ( - {/*Todo: remove the false to test, or when we feel confident with thsi approach */} - {Boolean(config.featureToggles.panelTitleSearch && !window.location.search?.includes('index=sql')) ? ( + {Boolean(config.featureToggles.panelTitleSearch) ? ( ; } diff --git a/public/app/features/search/page/components/ActionRow.tsx b/public/app/features/search/page/components/ActionRow.tsx index 7c309c11cff..091a057d2a2 100644 --- a/public/app/features/search/page/components/ActionRow.tsx +++ b/public/app/features/search/page/components/ActionRow.tsx @@ -9,8 +9,6 @@ import { TagFilter, TermCount } from 'app/core/components/TagFilter/TagFilter'; import { DashboardQuery, SearchLayout } from '../../types'; -import { getSortOptions } from './sorting'; - export const layoutOptions = [ { value: SearchLayout.Folders, icon: 'folder', ariaLabel: 'View by folders' }, { value: SearchLayout.List, icon: 'list-ul', ariaLabel: 'View as list' }, @@ -26,6 +24,7 @@ interface Props { onStarredFilterChange?: (event: FormEvent) => void; onTagFilterChange: (tags: string[]) => void; getTagOptions: () => Promise; + getSortOptions: () => Promise; onDatasourceChange: (ds?: string) => void; query: DashboardQuery; showStarredFilter?: boolean; @@ -54,6 +53,7 @@ export const ActionRow: FC = ({ onStarredFilterChange = () => {}, onTagFilterChange, getTagOptions, + getSortOptions, onDatasourceChange, query, showStarredFilter, diff --git a/public/app/features/search/page/components/FolderSection.tsx b/public/app/features/search/page/components/FolderSection.tsx index 39b508b17ab..167cc37afbc 100644 --- a/public/app/features/search/page/components/FolderSection.tsx +++ b/public/app/features/search/page/components/FolderSection.tsx @@ -3,9 +3,7 @@ import React, { FC } from 'react'; import { useAsync, useLocalStorage } from 'react-use'; import { GrafanaTheme } from '@grafana/data'; -import { getBackendSrv } from '@grafana/runtime'; import { Card, Checkbox, CollapsableSection, Icon, Spinner, stylesFactory, useTheme } from '@grafana/ui'; -import impressionSrv from 'app/core/services/impression_srv'; import { getSectionStorageKey } from 'app/features/search/utils'; import { useUniqueId } from 'app/plugins/datasource/influxdb/components/useUniqueId'; @@ -58,23 +56,14 @@ export const FolderSection: FC = ({ location: section.uid, sort: 'name_sort', }; - if (section.title === 'Starred') { + if (section.itemsUIDs) { query = { uid: section.itemsUIDs, // array of UIDs }; folderUid = undefined; folderTitle = undefined; - } else if (section.title === 'Recent') { - const ids = impressionSrv.getDashboardOpened(); - const uids = await getBackendSrv().get(`/api/dashboards/ids/${ids.slice(0, 30).join(',')}`); - if (uids?.length) { - query = { - uid: uids, - }; - } - folderUid = undefined; - folderTitle = undefined; } + const raw = await getGrafanaSearcher().search({ ...query, tags }); const v = raw.view.map( (item) => diff --git a/public/app/features/search/page/components/FolderView.tsx b/public/app/features/search/page/components/FolderView.tsx index 1ba64cffc0e..32d2efc458e 100644 --- a/public/app/features/search/page/components/FolderView.tsx +++ b/public/app/features/search/page/components/FolderView.tsx @@ -6,6 +6,8 @@ import { GrafanaTheme2 } from '@grafana/data'; import { selectors } from '@grafana/e2e-selectors'; import { getBackendSrv } from '@grafana/runtime'; import { Spinner, useStyles2 } from '@grafana/ui'; +import { contextSrv } from 'app/core/core'; +import impressionSrv from 'app/core/services/impression_srv'; import { GENERAL_FOLDER_UID } from '../../constants'; import { getGrafanaSearcher } from '../../service'; @@ -23,11 +25,20 @@ export const FolderView = ({ selection, selectionToggle, onTagSelected, tags, hi const results = useAsync(async () => { const folders: DashboardSection[] = []; if (!hidePseudoFolders) { - const stars = await getBackendSrv().get('api/user/stars'); - if (stars.length > 0) { - folders.push({ title: 'Starred', icon: 'star', kind: 'query-star', uid: '__starred', itemsUIDs: stars }); + if (contextSrv.isSignedIn) { + const stars = await getBackendSrv().get('api/user/stars'); + if (stars.length > 0) { + folders.push({ title: 'Starred', icon: 'star', kind: 'query-star', uid: '__starred', itemsUIDs: stars }); + } + } + + const ids = impressionSrv.getDashboardOpened(); + if (ids.length) { + const itemsUIDs = await getBackendSrv().get(`/api/dashboards/ids/${ids.slice(0, 30).join(',')}`); + if (itemsUIDs.length) { + folders.push({ title: 'Recent', icon: 'clock', kind: 'query-recent', uid: '__recent', itemsUIDs }); + } } - folders.push({ title: 'Recent', icon: 'clock', kind: 'query-recent', uid: '__recent' }); } folders.push({ title: 'General', url: '/dashboards', kind: 'folder', uid: GENERAL_FOLDER_UID }); diff --git a/public/app/features/search/page/components/SearchView.tsx b/public/app/features/search/page/components/SearchView.tsx index 43f540bf4ee..45e0ac95f3a 100644 --- a/public/app/features/search/page/components/SearchView.tsx +++ b/public/app/features/search/page/components/SearchView.tsx @@ -205,6 +205,7 @@ export const SearchView = ({ showManage, folderDTO, queryText, hidePseudoFolders onSortChange={onSortChange} onTagFilterChange={onTagFilterChange} getTagOptions={getTagOptions} + getSortOptions={getGrafanaSearcher().getSortOptions} onDatasourceChange={onDatasourceChange} query={query} /> diff --git a/public/app/features/search/page/components/columns.tsx b/public/app/features/search/page/components/columns.tsx index 2fd1c2860e9..b13f716db4b 100644 --- a/public/app/features/search/page/components/columns.tsx +++ b/public/app/features/search/page/components/columns.tsx @@ -3,7 +3,7 @@ import { isNumber } from 'lodash'; import React from 'react'; import SVG from 'react-inlinesvg'; -import { Field } from '@grafana/data'; +import { Field, getFieldDisplayName } from '@grafana/data'; import { config, getDataSourceSrv } from '@grafana/runtime'; import { Checkbox, Icon, IconButton, IconName, TagList } from '@grafana/ui'; @@ -11,7 +11,6 @@ import { QueryResponse, SearchResultMeta } from '../../service'; import { SelectionChecker, SelectionToggle } from '../selection'; import { TableColumn } from './SearchResultsTable'; -import { getSortFieldDisplayName } from './sorting'; const TYPE_COLUMN_WIDTH = 250; const DATASOURCE_COLUMN_WIDTH = 200; @@ -172,7 +171,7 @@ export const generateColumns = ( if (sortField) { columns.push({ - Header: () =>
{getSortFieldDisplayName(sortField.name)}
, + Header: () =>
{getFieldDisplayName(sortField)}
, Cell: (p) => { let value = sortField.values.get(p.row.index); try { @@ -182,7 +181,7 @@ export const generateColumns = ( } catch {} return (
- {value} + {`${value}`}
); }, diff --git a/public/app/features/search/page/components/sorting.ts b/public/app/features/search/page/components/sorting.ts deleted file mode 100644 index 0046d1b2b55..00000000000 --- a/public/app/features/search/page/components/sorting.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { SelectableValue } from '@grafana/data'; -import { config } from '@grafana/runtime'; - -// Enterprise only sort field values for dashboards -const sortFields = [ - { name: 'views_total', display: 'Views total' }, - { name: 'views_last_30_days', display: 'Views 30 days' }, - { name: 'errors_total', display: 'Errors total' }, - { name: 'errors_last_30_days', display: 'Errors 30 days' }, -]; - -// This should eventually be filled by an API call, but hardcoded is a good start -export async function getSortOptions(): Promise { - const opts: SelectableValue[] = [ - { value: 'name_sort', label: 'Alphabetically (A-Z)' }, - { value: '-name_sort', label: 'Alphabetically (Z-A)' }, - ]; - - if (config.licenseInfo.enabledFeatures.analytics) { - for (const sf of sortFields) { - opts.push({ value: `-${sf.name}`, label: `${sf.display} (most)` }); - opts.push({ value: `${sf.name}`, label: `${sf.display} (least)` }); - } - } - - return opts; -} - -/** Given the internal field name, this gives a reasonable display name for the table colum header */ -export function getSortFieldDisplayName(name: string) { - for (const sf of sortFields) { - if (sf.name === name) { - return sf.display; - } - } - return name; -} diff --git a/public/app/features/search/service/bluge.ts b/public/app/features/search/service/bluge.ts index 27814551442..38886ba369b 100644 --- a/public/app/features/search/service/bluge.ts +++ b/public/app/features/search/service/bluge.ts @@ -1,6 +1,6 @@ import { lastValueFrom } from 'rxjs'; -import { ArrayVector, DataFrame, DataFrameView, getDisplayProcessor } from '@grafana/data'; +import { ArrayVector, DataFrame, DataFrameView, getDisplayProcessor, SelectableValue } from '@grafana/data'; import { config, getDataSourceSrv } from '@grafana/runtime'; import { TermCount } from 'app/core/components/TagFilter/TagFilter'; import { GrafanaDatasource } from 'app/plugins/datasource/grafana/datasource'; @@ -16,10 +16,6 @@ export class BlugeSearcher implements GrafanaSearcher { return doSearchQuery(query); } - async list(location: string): Promise { - return doSearchQuery({ query: `list:${location ?? ''}` }); - } - async tags(query: SearchQuery): Promise { const ds = (await getDataSourceSrv().get('-- Grafana --')) as GrafanaDatasource; const target = { @@ -46,12 +42,29 @@ export class BlugeSearcher implements GrafanaSearcher { } return []; } + + // This should eventually be filled by an API call, but hardcoded is a good start + getSortOptions(): Promise { + const opts: SelectableValue[] = [ + { value: 'name_sort', label: 'Alphabetically (A-Z)' }, + { value: '-name_sort', label: 'Alphabetically (Z-A)' }, + ]; + + if (config.licenseInfo.enabledFeatures.analytics) { + for (const sf of sortFields) { + opts.push({ value: `-${sf.name}`, label: `${sf.display} (most)` }); + opts.push({ value: `${sf.name}`, label: `${sf.display} (least)` }); + } + } + + return Promise.resolve(opts); + } } const firstPageSize = 50; const nextPageSizes = 100; -export async function doSearchQuery(query: SearchQuery): Promise { +async function doSearchQuery(query: SearchQuery): Promise { const ds = (await getDataSourceSrv().get('-- Grafana --')) as GrafanaDatasource; const target = { ...query, @@ -84,8 +97,19 @@ export async function doSearchQuery(query: SearchQuery): Promise const meta = first.meta.custom as SearchResultMeta; if (!meta.locationInfo) { - meta.locationInfo = {}; + meta.locationInfo = {}; // always set it so we can append } + + // Set the field name to a better display name + if (meta.sortBy?.length) { + const field = first.fields.find((f) => f.name === meta.sortBy); + if (field) { + const name = getSortFieldDisplayName(field.name); + meta.sortBy = name; + field.name = name; // make it look nicer + } + } + const view = new DataFrameView(first); return { totalRows: meta.count ?? first.length, @@ -146,3 +170,21 @@ function getTermCountsFrom(frame: DataFrame): TermCount[] { } return counts; } + +// Enterprise only sort field values for dashboards +const sortFields = [ + { name: 'views_total', display: 'Views total' }, + { name: 'views_last_30_days', display: 'Views 30 days' }, + { name: 'errors_total', display: 'Errors total' }, + { name: 'errors_last_30_days', display: 'Errors 30 days' }, +]; + +/** Given the internal field name, this gives a reasonable display name for the table colum header */ +function getSortFieldDisplayName(name: string) { + for (const sf of sortFields) { + if (sf.name === name) { + return sf.display; + } + } + return name; +} diff --git a/public/app/features/search/service/searcher.ts b/public/app/features/search/service/searcher.ts index ff6675339d6..c61ed5bc552 100644 --- a/public/app/features/search/service/searcher.ts +++ b/public/app/features/search/service/searcher.ts @@ -1,11 +1,17 @@ +import { config } from '@grafana/runtime'; + import { BlugeSearcher } from './bluge'; +import { SQLSearcher } from './sql'; import { GrafanaSearcher } from './types'; let searcher: GrafanaSearcher | undefined = undefined; export function getGrafanaSearcher(): GrafanaSearcher { if (!searcher) { - searcher = new BlugeSearcher(); + const useBluge = + config.featureToggles.panelTitleSearch && // set in system configs + window.location.search.indexOf('index=sql') < 0; // or URL override + searcher = useBluge ? new BlugeSearcher() : new SQLSearcher(); } return searcher!; } diff --git a/public/app/features/search/service/sql.ts b/public/app/features/search/service/sql.ts new file mode 100644 index 00000000000..96e68ac6946 --- /dev/null +++ b/public/app/features/search/service/sql.ts @@ -0,0 +1,198 @@ +import { ArrayVector, DataFrame, DataFrameView, FieldType, getDisplayProcessor, SelectableValue } from '@grafana/data'; +import { config } from '@grafana/runtime'; +import { TermCount } from 'app/core/components/TagFilter/TagFilter'; +import { backendSrv } from 'app/core/services/backend_srv'; + +import { DashboardSearchHit } from '../types'; + +import { LocationInfo } from './types'; + +import { DashboardQueryResult, GrafanaSearcher, QueryResponse, SearchQuery } from '.'; + +interface APIQuery { + query?: string; + tag?: string[]; + limit?: number; + page?: number; + type?: string; + // DashboardIds []int64 + folderIds?: number[]; + sort?: string; + + // NEW!!!! TODO TODO: needs backend support? + dashboardUIDs?: string[]; +} + +// Internal object to hold folderId +interface LocationInfoEXT extends LocationInfo { + folderId?: number; +} + +export class SQLSearcher implements GrafanaSearcher { + locationInfo: Record = { + general: { + kind: 'folder', + name: 'General', + url: '/dashboards', + folderId: 0, + }, + }; // share location info with everyone + + async search(query: SearchQuery): Promise { + if (query.facet?.length) { + throw 'facets not supported!'; + } + const q: APIQuery = { + limit: 1000, // 1k max values + tag: query.tags, + sort: query.sort, + }; + + if (query.query === '*') { + if (query.kind?.length === 1 && query.kind[0] === 'folder') { + q.type = 'dash-folder'; + } + } else if (query.query?.length) { + q.query = query.query; + } + + if (query.uid) { + q.query = query.uid.join(', '); // TODO! this will return nothing + q.dashboardUIDs = query.uid; + } else if (query.location?.length) { + let info = this.locationInfo[query.location]; + if (!info) { + // This will load all folder folders + await this.doAPIQuery({ type: 'dash-folder', limit: 999 }); + info = this.locationInfo[query.location]; + } + q.folderIds = [info.folderId ?? 0]; + } + return this.doAPIQuery(q); + } + + // returns the appropriate sorting options + async getSortOptions(): Promise { + // { + // "sortOptions": [ + // { + // "description": "Sort results in an alphabetically ascending order", + // "displayName": "Alphabetically (A–Z)", + // "meta": "", + // "name": "alpha-asc" + // }, + // { + // "description": "Sort results in an alphabetically descending order", + // "displayName": "Alphabetically (Z–A)", + // "meta": "", + // "name": "alpha-desc" + // } + // ] + // } + const opts = await backendSrv.get('/api/search/sorting'); + return opts.sortOptions.map((v: any) => ({ + value: v.name, + label: v.displayName, + })); + } + + // NOTE: the bluge query will find tags within the current results, the SQL based one does not + async tags(query: SearchQuery): Promise { + const terms = (await backendSrv.get('/api/dashboards/tags')) as TermCount[]; + return terms.sort((a, b) => b.count - a.count); + } + + async doAPIQuery(query: APIQuery): Promise { + const rsp = (await backendSrv.get('/api/search', query)) as DashboardSearchHit[]; + + // Field values (columnar) + const kind: string[] = []; + const name: string[] = []; + const uid: string[] = []; + const url: string[] = []; + const tags: string[][] = []; + const location: string[] = []; + const sortBy: number[] = []; + let sortMetaName: string | undefined; + + for (let hit of rsp) { + const k = hit.type === 'dash-folder' ? 'folder' : 'dashboard'; + kind.push(k); + name.push(hit.title); + uid.push(hit.uid!); + url.push(hit.url); + tags.push(hit.tags); + sortBy.push(hit.sortMeta!); + + let v = hit.folderUid; + if (!v && k === 'dashboard') { + v = 'general'; + } + location.push(v!); + + if (hit.sortMetaName?.length) { + sortMetaName = hit.sortMetaName; + } + + if (hit.folderUid && hit.folderTitle) { + this.locationInfo[hit.folderUid] = { + kind: 'folder', + name: hit.folderTitle, + url: hit.folderUrl!, + folderId: hit.folderId, + }; + } else if (k === 'folder') { + this.locationInfo[hit.uid!] = { + kind: k, + name: hit.title!, + url: hit.url, + folderId: hit.id, + }; + } + } + + const data: DataFrame = { + fields: [ + { name: 'kind', type: FieldType.string, config: {}, values: new ArrayVector(kind) }, + { name: 'name', type: FieldType.string, config: {}, values: new ArrayVector(name) }, + { name: 'uid', type: FieldType.string, config: {}, values: new ArrayVector(uid) }, + { name: 'url', type: FieldType.string, config: {}, values: new ArrayVector(url) }, + { name: 'tags', type: FieldType.other, config: {}, values: new ArrayVector(tags) }, + { name: 'location', type: FieldType.string, config: {}, values: new ArrayVector(location) }, + ], + length: name.length, + meta: { + custom: { + count: name.length, + max_score: 1, + locationInfo: this.locationInfo, + }, + }, + }; + + // Add enterprise sort fields as a field in the frame + if (sortMetaName?.length && sortBy.length) { + data.meta!.custom!.sortBy = sortMetaName; + data.fields.push({ + name: sortMetaName, // Used in display + type: FieldType.number, + config: {}, + values: new ArrayVector(sortBy), + }); + } + + for (const field of data.fields) { + field.display = getDisplayProcessor({ field, theme: config.theme2 }); + } + + const view = new DataFrameView(data); + return { + totalRows: data.length, + view, + + // Paging not supported with this version + loadMoreItems: async (startIndex: number, stopIndex: number): Promise => {}, + isItemLoaded: (index: number): boolean => true, + }; + } +} diff --git a/public/app/features/search/service/types.ts b/public/app/features/search/service/types.ts index adff98d3467..1b0d79b9d2a 100644 --- a/public/app/features/search/service/types.ts +++ b/public/app/features/search/service/types.ts @@ -1,4 +1,4 @@ -import { DataFrameView } from '@grafana/data'; +import { DataFrameView, SelectableValue } from '@grafana/data'; import { TermCount } from 'app/core/components/TagFilter/TagFilter'; export interface FacetField { @@ -32,7 +32,6 @@ export interface DashboardQueryResult { tags: string[]; location: string; // url that can be split ds_uid: string[]; - score?: number; } export interface LocationInfo { @@ -45,6 +44,7 @@ export interface SearchResultMeta { count: number; max_score: number; locationInfo: Record; + sortBy?: string; } export interface QueryResponse { @@ -62,6 +62,6 @@ export interface QueryResponse { export interface GrafanaSearcher { search: (query: SearchQuery) => Promise; - list: (location: string) => Promise; tags: (query: SearchQuery) => Promise; + getSortOptions: () => Promise; }