Datasource/Cloudwatch: Adds support for Cloudwatch Logs (#23566)

* Datasource/Cloudwatch: Adds support for Cloudwatch Logs

* Fix rebase leftover

* Use jsurl for AWS url serialization

* WIP: Temporary workaround for CLIQ metrics

* Only allow up to 20 log groups to be selected

* WIP additional changes

* More changes based on feedback

* More changes based on PR feedback

* Fix strict null errors
This commit is contained in:
kay delaney
2020-04-25 22:48:20 +02:00
committed by GitHub
parent af00fa7214
commit f48ba11d4c
78 changed files with 3914 additions and 399 deletions
@@ -1,4 +1,5 @@
import { DataFrame, FieldType, Field, Vector } from '../types';
import {
Table,
ArrowType,
@@ -162,15 +163,18 @@ export function grafanaDataFrameToArrowTable(data: DataFrame): Table {
}
export function resultsToDataFrames(rsp: any): DataFrame[] {
const frames: DataFrame[] = [];
for (const res of Object.values(rsp.results)) {
const r = res as any;
if (r.dataframes) {
for (const b of r.dataframes) {
const t = base64StringToArrowTable(b as string);
frames.push(arrowTableToDataFrame(t));
}
}
if (rsp === undefined || rsp.results === undefined) {
return [];
}
const results = rsp.results as Array<{ dataframes: string[] }>;
const frames: DataFrame[] = Object.values(results).flatMap(res => {
if (!res.dataframes) {
return [];
}
return res.dataframes.map((b: string) => arrowTableToDataFrame(base64StringToArrowTable(b)));
});
return frames;
}
@@ -58,11 +58,9 @@ export class FieldCache {
}
getFirstFieldOfType(type: FieldType): FieldWithIndex | undefined {
const arr = this.fieldByType[type];
if (arr && arr.length > 0) {
return arr[0];
}
return undefined;
const fields = this.fieldByType[type];
const firstField = fields.find(field => !(field.config.custom && field.config.custom['Hidden']));
return firstField;
}
hasFieldNamed(name: string): boolean {
@@ -178,6 +178,10 @@ export function guessFieldTypeFromNameAndValue(name: string, v: any): FieldType
* TODO: better Date/Time support! Look for standard date strings?
*/
export function guessFieldTypeFromValue(v: any): FieldType {
if (v instanceof Date || isDateTime(v)) {
return FieldType.time;
}
if (isNumber(v)) {
return FieldType.number;
}
@@ -198,10 +202,6 @@ export function guessFieldTypeFromValue(v: any): FieldType {
return FieldType.boolean;
}
if (v instanceof Date || isDateTime(v)) {
return FieldType.time;
}
return FieldType.other;
}
@@ -230,17 +230,19 @@ export function guessFieldTypeForField(field: Field): FieldType | undefined {
}
/**
* @returns a copy of the series with the best guess for each field type
* If the series already has field types defined, they will be used
* @returns A copy of the series with the best guess for each field type.
* If the series already has field types defined, they will be used, unless `guessDefined` is true.
* @param series The DataFrame whose field's types should be guessed
* @param guessDefined Whether to guess types of fields with already defined types
*/
export const guessFieldTypes = (series: DataFrame): DataFrame => {
for (let i = 0; i < series.fields.length; i++) {
if (!series.fields[i].type) {
export const guessFieldTypes = (series: DataFrame, guessDefined = false): DataFrame => {
for (const field of series.fields) {
if (!field.type || field.type === FieldType.other || guessDefined) {
// Something is missing a type, return a modified copy
return {
...series,
fields: series.fields.map(field => {
if (field.type && field.type !== FieldType.other) {
if (field.type && field.type !== FieldType.other && !guessDefined) {
return field;
}
// Calculate a reasonable schema value
@@ -38,6 +38,7 @@ export interface FeatureToggles {
* Available only in Grafana Enterprise
*/
meta: boolean;
cloudwatchLogs: boolean;
newVariables: boolean;
}
@@ -274,6 +274,8 @@ export abstract class DataSourceApi<
getVersion?(optionalOptions?: any): Promise<string>;
showContextToggle?(row?: LogRowModel): boolean;
/**
* Can be optionally implemented to allow datasource to be a source of annotations for dashboard. To be visible
* in the annotation editor `annotations` capability also needs to be enabled in plugin.json.
@@ -307,6 +309,8 @@ export interface QueryEditorProps<
* Contains query response filtered by refId of QueryResultBase and possible query error
*/
data?: PanelData;
exploreMode?: ExploreMode;
exploreId?: any;
}
export enum DataSourceStatus {
@@ -329,6 +333,7 @@ export interface ExploreQueryFieldProps<
onBlur?: () => void;
absoluteRange?: AbsoluteTimeRange;
exploreMode?: ExploreMode;
exploreId?: any;
}
export interface ExploreStartPageProps {
+1
View File
@@ -52,6 +52,7 @@ export class GrafanaBootConfig implements GrafanaConfig {
expressions: false,
newEdit: false,
meta: false,
cloudwatchLogs: false,
newVariables: true,
};
licenseInfo: LicenseInfo = {} as LicenseInfo;
@@ -10,7 +10,7 @@ interface Props {
isFocused?: boolean;
isInvalid?: boolean;
tooltip?: PopoverContent;
width?: number;
width?: number | 'auto';
}
export const FormLabel: FunctionComponent<Props> = ({
@@ -23,7 +23,7 @@ export const FormLabel: FunctionComponent<Props> = ({
width,
...rest
}) => {
const classes = classNames(`gf-form-label width-${width ? width : '10'}`, className, {
const classes = classNames(className, `gf-form-label width-${width ? width : '10'}`, {
'gf-form-label--is-focused': isFocused,
'gf-form-label--is-invalid': isInvalid,
});
@@ -8,6 +8,7 @@ import {
LogRowContextQueryErrors,
HasMoreContextRows,
LogRowContextProvider,
RowContextOptions,
} from './LogRowContextProvider';
import { Themeable } from '../../types/theme';
import { withTheme } from '../../themes/index';
@@ -33,8 +34,9 @@ interface Props extends Themeable {
onClickFilterLabel?: (key: string, value: string) => void;
onClickFilterOutLabel?: (key: string, value: string) => void;
onContextClick?: () => void;
getRowContext: (row: LogRowModel, options?: any) => Promise<DataQueryResponse>;
getRowContext: (row: LogRowModel, options?: RowContextOptions) => Promise<DataQueryResponse>;
getFieldLinks?: (field: Field, rowIndex: number) => Array<LinkModel<Field>>;
showContextToggle?: (row?: LogRowModel) => boolean;
}
interface State {
@@ -122,6 +124,7 @@ class UnThemedLogRow extends PureComponent<Props, State> {
row,
showDuplicates,
timeZone,
showContextToggle,
showLabels,
showTime,
wrapLogMessage,
@@ -176,7 +179,8 @@ class UnThemedLogRow extends PureComponent<Props, State> {
hasMoreContextRows={hasMoreContextRows}
updateLimit={updateLimit}
context={context}
showContext={showContext}
contextIsOpen={showContext}
showContextToggle={showContextToggle}
wrapLogMessage={wrapLogMessage}
onToggleContext={this.toggleContext}
/>
@@ -5,6 +5,11 @@ import useAsync from 'react-use/lib/useAsync';
import { DataQueryResponse, DataQueryError } from '@grafana/data';
export interface RowContextOptions {
direction?: 'BACKWARD' | 'FORWARD';
limit?: number;
}
export interface LogRowContextRows {
before?: string[];
after?: string[];
@@ -26,7 +31,7 @@ interface ResultType {
interface LogRowContextProviderProps {
row: LogRowModel;
getRowContext: (row: LogRowModel, options?: any) => Promise<DataQueryResponse>;
getRowContext: (row: LogRowModel, options?: RowContextOptions) => Promise<DataQueryResponse>;
children: (props: {
result: LogRowContextRows;
errors: LogRowContextQueryErrors;
@@ -36,7 +41,7 @@ interface LogRowContextProviderProps {
}
export const getRowContexts = async (
getRowContext: (row: LogRowModel, options?: any) => Promise<DataQueryResponse>,
getRowContext: (row: LogRowModel, options?: RowContextOptions) => Promise<DataQueryResponse>,
row: LogRowModel,
limit: number
) => {
@@ -20,10 +20,11 @@ import { LogMessageAnsi } from './LogMessageAnsi';
interface Props extends Themeable {
row: LogRowModel;
hasMoreContextRows?: HasMoreContextRows;
showContext: boolean;
contextIsOpen: boolean;
wrapLogMessage: boolean;
errors?: LogRowContextQueryErrors;
context?: LogRowContextRows;
showContextToggle?: (row?: LogRowModel) => boolean;
highlighterExpressions?: string[];
getRows: () => LogRowModel[];
onToggleContext: () => void;
@@ -74,7 +75,8 @@ class UnThemedLogRowMessage extends PureComponent<Props> {
hasMoreContextRows,
updateLimit,
context,
showContext,
contextIsOpen,
showContextToggle,
wrapLogMessage,
onToggleContext,
} = this.props;
@@ -97,7 +99,7 @@ class UnThemedLogRowMessage extends PureComponent<Props> {
return (
<td className={style.logsRowMessage}>
<div className={cx(styles.positionRelative, { [styles.horizontalScroll]: !wrapLogMessage })}>
{showContext && context && (
{contextIsOpen && context && (
<LogRowContext
row={row}
context={context}
@@ -111,7 +113,7 @@ class UnThemedLogRowMessage extends PureComponent<Props> {
}}
/>
)}
<span className={cx(styles.positionRelative, { [styles.rowWithContext]: showContext })}>
<span className={cx(styles.positionRelative, { [styles.rowWithContext]: contextIsOpen })}>
{needsHighlighter ? (
<Highlighter
style={whiteSpacePreWrap}
@@ -126,9 +128,9 @@ class UnThemedLogRowMessage extends PureComponent<Props> {
entry
)}
</span>
{row.searchWords && row.searchWords.length > 0 && (
{showContextToggle?.(row) && (
<span onClick={this.onContextToggle} className={cx(style.context)}>
{showContext ? 'Hide' : 'Show'} context
{contextIsOpen ? 'Hide' : 'Show'} context
</span>
)}
</div>
@@ -8,6 +8,7 @@ import { getLogRowStyles } from './getLogRowStyles';
//Components
import { LogRow } from './LogRow';
import { RowContextOptions } from './LogRowContextProvider';
export const PREVIEW_LIMIT = 100;
export const RENDER_LIMIT = 500;
@@ -17,6 +18,7 @@ export interface Props extends Themeable {
deduplicatedRows?: LogRowModel[];
dedupStrategy: LogsDedupStrategy;
highlighterExpressions?: string[];
showContextToggle?: (row?: LogRowModel) => boolean;
showLabels: boolean;
showTime: boolean;
wrapLogMessage: boolean;
@@ -26,7 +28,7 @@ export interface Props extends Themeable {
previewLimit?: number;
onClickFilterLabel?: (key: string, value: string) => void;
onClickFilterOutLabel?: (key: string, value: string) => void;
getRowContext?: (row: LogRowModel, options?: any) => Promise<any>;
getRowContext?: (row: LogRowModel, options?: RowContextOptions) => Promise<any>;
getFieldLinks?: (field: Field, rowIndex: number) => Array<LinkModel<Field>>;
}
@@ -72,6 +74,7 @@ class UnThemedLogRows extends PureComponent<Props, State> {
render() {
const {
dedupStrategy,
showContextToggle,
showLabels,
showTime,
wrapLogMessage,
@@ -119,6 +122,7 @@ class UnThemedLogRows extends PureComponent<Props, State> {
getRowContext={getRowContext}
highlighterExpressions={highlighterExpressions}
row={row}
showContextToggle={showContextToggle}
showDuplicates={showDuplicates}
showLabels={showLabels}
showTime={showTime}
@@ -138,6 +142,7 @@ class UnThemedLogRows extends PureComponent<Props, State> {
getRows={getRows}
getRowContext={getRowContext}
row={row}
showContextToggle={showContextToggle}
showDuplicates={showDuplicates}
showLabels={showLabels}
showTime={showTime}
@@ -29,6 +29,7 @@ export interface QueryFieldProps {
onRunQuery?: () => void;
onBlur?: () => void;
onChange?: (value: string) => void;
onClick?: (event: Event, editor: CoreEditor, next: () => any) => any;
onTypeahead?: (typeahead: TypeaheadInput) => Promise<TypeaheadOutput>;
onWillApplySuggestion?: (suggestion: string, state: SuggestionsState) => string;
placeholder?: string;
@@ -167,6 +168,7 @@ export class QueryField extends React.PureComponent<QueryFieldProps, QueryFieldS
*/
handleBlur = (event: Event, editor: CoreEditor, next: Function) => {
const { onBlur } = this.props;
if (onBlur) {
onBlur();
} else {
@@ -196,6 +198,7 @@ export class QueryField extends React.PureComponent<QueryFieldProps, QueryFieldS
autoCorrect={false}
readOnly={this.props.disabled}
onBlur={this.handleBlur}
onClick={this.props.onClick}
// onKeyDown={this.onKeyDown}
onChange={(change: { value: Value }) => {
this.onChange(change.value, false);
@@ -91,6 +91,7 @@ export function SelectBase<T>({
allowCustomValue = false,
autoFocus = false,
backspaceRemovesValue = true,
cacheOptions,
className,
closeMenuOnSelect = true,
components,
@@ -106,6 +107,7 @@ export function SelectBase<T>({
isLoading = false,
isMulti = false,
isOpen,
isOptionDisabled,
isSearchable = true,
loadOptions,
loadingMessage = 'Loading options...',
@@ -183,6 +185,7 @@ export function SelectBase<T>({
isDisabled: disabled,
isLoading,
isMulti,
isOptionDisabled,
isSearchable,
maxMenuHeight,
maxVisibleValues,
@@ -217,6 +220,7 @@ export function SelectBase<T>({
ReactSelectComponent = allowCustomValue ? AsyncCreatable : ReactAsyncSelect;
asyncSelectProps = {
loadOptions,
cacheOptions,
defaultOptions,
};
}
@@ -337,6 +341,10 @@ export function SelectBase<T>({
position: 'relative',
width: width ? `${8 * width}px` : '100%',
}),
option: (provided: any, state: any) => ({
...provided,
opacity: state.isDisabled ? 0.5 : 1,
}),
}}
className={className}
{...commonSelectProps}
@@ -19,6 +19,7 @@ export interface SelectCommonProps<T> {
getOptionLabel?: (item: SelectableValue<T>) => string;
getOptionValue?: (item: SelectableValue<T>) => string;
inputValue?: string;
invalid?: boolean;
isClearable?: boolean;
isLoading?: boolean;
isMulti?: boolean;
@@ -51,6 +52,7 @@ export interface SelectCommonProps<T> {
value?: SelectValue<T>;
/** Sets the width to a multiple of 8px. Should only be used with inline forms. Setting width of the container is preferred in other cases.*/
width?: number;
isOptionDisabled?: () => boolean;
}
export interface SelectAsyncProps<T> {
@@ -58,6 +60,8 @@ export interface SelectAsyncProps<T> {
defaultOptions?: boolean | Array<SelectableValue<T>>;
/** Asynchronously load select options */
loadOptions?: (query: string) => Promise<Array<SelectableValue<T>>>;
/** If cacheOptions is true, then the loaded data will be cached. The cache will remain until cacheOptions changes value. */
cacheOptions?: boolean;
/** Message to display when options are loading */
loadingMessage?: string;
}
@@ -8,6 +8,7 @@ const getStyles = (theme: GrafanaTheme, height: number, visible: boolean) => {
return {
typeaheadItem: css`
label: type-ahead-item;
z-index: 11;
padding: ${theme.spacing.sm} ${theme.spacing.sm} ${theme.spacing.sm} ${theme.spacing.md};
border-radius: ${theme.border.radius.md};
border: ${selectThemeVariant(
@@ -28,7 +28,7 @@ const getStyles = (theme: GrafanaTheme) => ({
font-size: ${theme.typography.size.sm};
text-overflow: ellipsis;
overflow: hidden;
z-index: 1;
z-index: 11;
display: block;
white-space: nowrap;
cursor: pointer;
@@ -5,5 +5,5 @@ export { IndentationPlugin } from './indentation';
export { NewlinePlugin } from './newline';
export { RunnerPlugin } from './runner';
export { SelectionShortcutsPlugin } from './selection_shortcuts';
export { SlatePrism } from './slate-prism';
export { SlatePrism, Token } from './slate-prism';
export { SuggestionsPlugin } from './suggestions';
@@ -4,6 +4,18 @@ import { Plugin } from '@grafana/slate-react';
import Options, { OptionsFormat } from './options';
import TOKEN_MARK from './TOKEN_MARK';
export interface Token {
content: string;
offsets?: {
start: number;
end: number;
};
types: string[];
aliases: string[];
prev?: Token | null;
next?: Token | null;
}
/**
* A Slate plugin to highlight code syntax.
*/
@@ -15,7 +27,25 @@ export function SlatePrism(optsParam: OptionsFormat = {}): Plugin {
if (!opts.onlyIn(node)) {
return next();
}
return decorateNode(opts, Block.create(node as Block));
const block = Block.create(node as Block);
const grammarName = opts.getSyntax(block);
const grammar = Prism.languages[grammarName];
if (!grammar) {
// Grammar not loaded
return [];
}
// Tokenize the whole block text
const texts = block.getTexts();
const blockText = texts.map(text => text && text.getText()).join('\n');
const tokens = Prism.tokenize(blockText, grammar);
const flattened = flattenTokens(tokens);
// @ts-ignore
editor.setData({ tokens: flattened });
return decorateNode(opts, tokens, block);
},
renderDecoration: (props, editor, next) =>
@@ -33,18 +63,8 @@ export function SlatePrism(optsParam: OptionsFormat = {}): Plugin {
/**
* Returns the decoration for a node
*/
function decorateNode(opts: Options, block: Block) {
const grammarName = opts.getSyntax(block);
const grammar = Prism.languages[grammarName];
if (!grammar) {
// Grammar not loaded
return [];
}
// Tokenize the whole block text
function decorateNode(opts: Options, tokens: Array<string | Prism.Token>, block: Block) {
const texts = block.getTexts();
const blockText = texts.map(text => text && text.getText()).join('\n');
const tokens = Prism.tokenize(blockText, grammar);
// The list of decorations to return
const decorations: Decoration[] = [];
@@ -67,13 +87,17 @@ function decorateNode(opts: Options, block: Block) {
className: `prism-token token ${accu}`,
block,
});
if (decoration) {
decorations.push(decoration);
}
}
offset += token.length;
} else {
accu = `${accu} ${token.type} ${token.alias || ''}`;
accu = `${accu} ${token.type}`;
if (token.alias) {
accu += ' ' + token.alias;
}
if (typeof token.content === 'string') {
const decoration = createDecoration({
@@ -85,6 +109,7 @@ function decorateNode(opts: Options, block: Block) {
className: `prism-token token ${accu}`,
block,
});
if (decoration) {
decorations.push(decoration);
}
@@ -158,3 +183,71 @@ function createDecoration({
return myDec;
}
function flattenToken(token: string | Prism.Token | Array<string | Prism.Token>): Token[] {
if (typeof token === 'string') {
return [
{
content: token,
types: [],
aliases: [],
},
];
} else if (Array.isArray(token)) {
return token.flatMap(t => flattenToken(t));
} else if (token instanceof Prism.Token) {
return flattenToken(token.content).flatMap(t => {
let aliases: string[] = [];
if (typeof token.alias === 'string') {
aliases = [token.alias];
} else {
aliases = token.alias ?? [];
}
return {
content: t.content,
types: [token.type, ...t.types],
aliases: [...aliases, ...t.aliases],
};
});
}
return [];
}
export function flattenTokens(token: string | Prism.Token | Array<string | Prism.Token>) {
const tokens = flattenToken(token);
if (!tokens.length) {
return [];
}
const firstToken = tokens[0];
firstToken.prev = null;
firstToken.next = tokens.length >= 2 ? tokens[1] : null;
firstToken.offsets = {
start: 0,
end: firstToken.content.length,
};
for (let i = 1; i < tokens.length - 1; i++) {
tokens[i].prev = tokens[i - 1];
tokens[i].next = tokens[i + 1];
tokens[i].offsets = {
start: tokens[i - 1].offsets!.end,
end: tokens[i - 1].offsets!.end + tokens[i].content.length,
};
}
const lastToken = tokens[tokens.length - 1];
lastToken.prev = tokens.length >= 2 ? tokens[tokens.length - 2] : null;
lastToken.next = null;
lastToken.offsets = {
start: tokens.length >= 2 ? tokens[tokens.length - 2].offsets!.end : 0,
end:
tokens.length >= 2 ? tokens[tokens.length - 2].offsets!.end + lastToken.content.length : lastToken.content.length,
};
return tokens;
}
@@ -217,14 +217,16 @@ const handleTypeahead = async (
// Get decorations associated with the current line
const parentBlock = value.document.getClosestBlock(value.focusBlock.key);
const myOffset = value.selection.start.offset - 1;
const selectionStartOffset = value.selection.start.offset - 1;
const decorations = parentBlock && parentBlock.getDecorations(editor as any);
const filteredDecorations = decorations
? decorations
.filter(
decoration =>
decoration!.start.offset <= myOffset && decoration!.end.offset > myOffset && decoration!.type === TOKEN_MARK
decoration!.start.offset <= selectionStartOffset &&
decoration!.end.offset > selectionStartOffset &&
decoration!.type === TOKEN_MARK
)
.toArray()
: [];
@@ -235,7 +237,7 @@ const handleTypeahead = async (
decorations
.filter(
decoration =>
decoration!.end.offset <= myOffset &&
decoration!.end.offset <= selectionStartOffset &&
decoration!.type === TOKEN_MARK &&
decoration!.data.get('className').includes('label-key')
)
@@ -272,6 +274,7 @@ const handleTypeahead = async (
value,
wrapperClasses,
labelKey: labelKey || undefined,
editor,
});
const filteredSuggestions = suggestions
@@ -280,28 +283,29 @@ const handleTypeahead = async (
return group;
}
let newGroup = { ...group };
if (prefix) {
// Filter groups based on prefix
if (!group.skipFilter) {
group.items = group.items.filter(c => (c.filterText || c.label).length >= prefix.length);
newGroup.items = newGroup.items.filter(c => (c.filterText || c.label).length >= prefix.length);
if (group.prefixMatch) {
group.items = group.items.filter(c => (c.filterText || c.label).startsWith(prefix));
newGroup.items = newGroup.items.filter(c => (c.filterText || c.label).startsWith(prefix));
} else {
group.items = group.items.filter(c => (c.filterText || c.label).includes(prefix));
newGroup.items = newGroup.items.filter(c => (c.filterText || c.label).includes(prefix));
}
}
// Filter out the already typed value (prefix) unless it inserts custom text
group.items = group.items.filter(c => c.insertText || (c.filterText || c.label) !== prefix);
newGroup.items = newGroup.items.filter(c => c.insertText || (c.filterText || c.label) !== prefix);
}
if (!group.skipSort) {
group.items = sortBy(group.items, (item: CompletionItem) => item.sortText || item.label);
newGroup.items = sortBy(newGroup.items, (item: CompletionItem) => item.sortText || item.label);
}
return group;
return newGroup;
})
.filter(group => group.items && group.items.length); // Filter out empty groups
.filter(gr => gr.items && gr.items.length); // Filter out empty groups
onStateChange({
groupedItems: filteredSuggestions,
+2 -3
View File
@@ -1,5 +1,4 @@
import { Value } from 'slate';
import { Editor } from '@grafana/slate-react';
import { Value, Editor as CoreEditor } from 'slate';
export interface CompletionItemGroup {
/**
@@ -98,7 +97,7 @@ export interface TypeaheadInput {
wrapperClasses: string[];
labelKey?: string;
value?: Value;
editor?: Editor;
editor?: CoreEditor;
}
export interface SuggestionsState {