From eba74f04080906d78e5a670ab5dda8e02ad9608f Mon Sep 17 00:00:00 2001 From: Ivan Ortega Alba Date: Fri, 22 Sep 2023 16:03:50 +0200 Subject: [PATCH] Analytics: Track usage of auto-generate functionality (#75267) --- .../components/GenAI/GenAIButton.test.tsx | 28 +++++++--- .../components/GenAI/GenAIButton.tsx | 21 +++++--- .../GenAI/GenAIDashDescriptionButton.tsx | 6 ++- .../components/GenAI/GenAIDashTitleButton.tsx | 4 +- .../GenAI/GenAIDashboardChangesButton.tsx | 10 +++- .../GenAI/GenAIPanelDescriptionButton.tsx | 52 +++++++++++-------- .../GenAI/GenAIPanelTitleButton.tsx | 50 ++++++++++-------- .../dashboard/components/GenAI/tracking.ts | 13 +++++ .../dashboard/components/GenAI/utils.ts | 2 +- 9 files changed, 121 insertions(+), 65 deletions(-) create mode 100644 public/app/features/dashboard/components/GenAI/tracking.ts diff --git a/public/app/features/dashboard/components/GenAI/GenAIButton.test.tsx b/public/app/features/dashboard/components/GenAI/GenAIButton.test.tsx index 22f0651c0fd..37f8c4c428b 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIButton.test.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIButton.test.tsx @@ -16,9 +16,9 @@ jest.mock('./utils', () => ({ })); describe('GenAIButton', () => { - const onReply = jest.fn(); + const onGenerate = jest.fn(); - function setup(props: GenAIButtonProps = { onReply, messages: [] }) { + function setup(props: GenAIButtonProps = { onGenerate, messages: [] }) { return render( @@ -99,23 +99,23 @@ describe('GenAIButton', () => { replyHandler('Generated text', isDoneGeneratingMessage); return new Promise(() => new Subscription()); }); - const onReply = jest.fn(); - setup({ onReply, messages: [] }); + const onGenerate = jest.fn(); + setup({ onGenerate, messages: [] }); const generateButton = await screen.findByRole('button'); // Click the button await fireEvent.click(generateButton); await waitFor(() => expect(generateButton).toBeEnabled()); - await waitFor(() => expect(onReply).toHaveBeenCalledTimes(1)); + await waitFor(() => expect(onGenerate).toHaveBeenCalledTimes(1)); // Wait for the loading state to be resolved - expect(onReply).toHaveBeenCalledTimes(1); + expect(onGenerate).toHaveBeenCalledTimes(1); }); it('should call the LLM service with the messages configured and the right temperature', async () => { - const onReply = jest.fn(); + const onGenerate = jest.fn(); const messages = [{ content: 'Generate X', role: 'system' as Role }]; - setup({ onReply, messages, temperature: 3 }); + setup({ onGenerate, messages, temperature: 3 }); const generateButton = await screen.findByRole('button'); await fireEvent.click(generateButton); @@ -123,5 +123,17 @@ describe('GenAIButton', () => { await waitFor(() => expect(generateTextWithLLM).toHaveBeenCalledTimes(1)); await waitFor(() => expect(generateTextWithLLM).toHaveBeenCalledWith(messages, expect.any(Function), 3)); }); + + it('should call the onClick callback', async () => { + const onGenerate = jest.fn(); + const onClick = jest.fn(); + const messages = [{ content: 'Generate X', role: 'system' as Role }]; + setup({ onGenerate, messages, temperature: 3, onClick }); + + const generateButton = await screen.findByRole('button'); + await fireEvent.click(generateButton); + + await waitFor(() => expect(onClick).toHaveBeenCalledTimes(1)); + }); }); }); diff --git a/public/app/features/dashboard/components/GenAI/GenAIButton.tsx b/public/app/features/dashboard/components/GenAI/GenAIButton.tsx index 09ca52936da..e0a36c12a37 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIButton.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIButton.tsx @@ -7,20 +7,27 @@ import { Button, Spinner, useStyles2, Link, Tooltip } from '@grafana/ui'; import { Message, generateTextWithLLM, isLLMPluginEnabled } from './utils'; export interface GenAIButtonProps { + // Button label text text?: string; + // Button label text when loading loadingText?: string; + // Button click handler onClick?: (e: React.MouseEvent) => void; + // Messages to send to the LLM plugin messages: Message[]; - onReply: (response: string, isDone: boolean) => void; + // Callback when the LLM plugin responds. It is sreaming, so it will be called multiple times. + onGenerate: (response: string, isDone: boolean) => void; + // Temperature for the LLM plugin. Default is 1. + // Closer to 0 means more conservative, closer to 1 means more creative. temperature?: number; } export const GenAIButton = ({ text = 'Auto-generate', loadingText = 'Generating', - onClick, + onClick: onClickProp, messages, - onReply, + onGenerate, temperature = 1, }: GenAIButtonProps) => { const styles = useStyles2(getStyles); @@ -29,11 +36,11 @@ export const GenAIButton = ({ const replyHandler = (response: string, isDone: boolean) => { setLoading(!isDone); - onReply(response, isDone); + onGenerate(response, isDone); }; - const onGenerate = (e: React.MouseEvent) => { - onClick?.(e); + const onClick = (e: React.MouseEvent) => { + onClickProp?.(e); setLoading(true); generateTextWithLLM(messages, replyHandler, temperature); }; @@ -67,7 +74,7 @@ export const GenAIButton = ({ } > - diff --git a/public/app/features/dashboard/components/GenAI/GenAIDashDescriptionButton.tsx b/public/app/features/dashboard/components/GenAI/GenAIDashDescriptionButton.tsx index 2103a7f3fb1..ba1fd95762a 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIDashDescriptionButton.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIDashDescriptionButton.tsx @@ -3,6 +3,7 @@ import React from 'react'; import { DashboardModel } from '../../state'; import { GenAIButton } from './GenAIButton'; +import { EventSource, reportGenerateAIButtonClicked } from './tracking'; import { Message, Role } from './utils'; interface GenAIDashDescriptionButtonProps { @@ -17,8 +18,11 @@ const DESCRIPTION_GENERATION_STANDARD_PROMPT = export const GenAIDashDescriptionButton = ({ onGenerate, dashboard }: GenAIDashDescriptionButtonProps) => { const messages = React.useMemo(() => getMessages(dashboard), [dashboard]); + const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.dashboardDescription), []); - return ; + return ( + + ); }; function getMessages(dashboard: DashboardModel): Message[] { diff --git a/public/app/features/dashboard/components/GenAI/GenAIDashTitleButton.tsx b/public/app/features/dashboard/components/GenAI/GenAIDashTitleButton.tsx index ff27793317a..4dc30855589 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIDashTitleButton.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIDashTitleButton.tsx @@ -3,6 +3,7 @@ import React from 'react'; import { DashboardModel } from '../../state'; import { GenAIButton } from './GenAIButton'; +import { EventSource, reportGenerateAIButtonClicked } from './tracking'; import { Message, Role } from './utils'; interface GenAIDashTitleButtonProps { @@ -17,8 +18,9 @@ const DESCRIPTION_GENERATION_STANDARD_PROMPT = export const GenAIDashTitleButton = ({ onGenerate, dashboard }: GenAIDashTitleButtonProps) => { const messages = React.useMemo(() => getMessages(dashboard), [dashboard]); + const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.dashboardTitle), []); - return ; + return ; }; function getMessages(dashboard: DashboardModel): Message[] { diff --git a/public/app/features/dashboard/components/GenAI/GenAIDashboardChangesButton.tsx b/public/app/features/dashboard/components/GenAI/GenAIDashboardChangesButton.tsx index f46266aac44..e6091090e58 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIDashboardChangesButton.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIDashboardChangesButton.tsx @@ -3,6 +3,7 @@ import React, { useMemo } from 'react'; import { DashboardModel } from '../../state'; import { GenAIButton } from './GenAIButton'; +import { EventSource, reportGenerateAIButtonClicked } from './tracking'; import { getDashboardChanges, Message, Role } from './utils'; interface GenAIDashboardChangesButtonProps { @@ -26,9 +27,16 @@ const CHANGES_GENERATION_STANDARD_PROMPT = [ export const GenAIDashboardChangesButton = ({ dashboard, onGenerate }: GenAIDashboardChangesButtonProps) => { const messages = useMemo(() => getMessages(dashboard), [dashboard]); + const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.dashboardChanges), []); return ( - + ); }; diff --git a/public/app/features/dashboard/components/GenAI/GenAIPanelDescriptionButton.tsx b/public/app/features/dashboard/components/GenAI/GenAIPanelDescriptionButton.tsx index c835eaa3683..fc17e0e13b0 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIPanelDescriptionButton.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIPanelDescriptionButton.tsx @@ -4,6 +4,7 @@ import { getDashboardSrv } from '../../services/DashboardSrv'; import { PanelModel } from '../../state'; import { GenAIButton } from './GenAIButton'; +import { EventSource, reportGenerateAIButtonClicked } from './tracking'; import { Message, Role } from './utils'; interface GenAIPanelDescriptionButtonProps { @@ -17,28 +18,33 @@ const DESCRIPTION_GENERATION_STANDARD_PROMPT = 'The description should be shorter than 140 characters.'; export const GenAIPanelDescriptionButton = ({ onGenerate, panel }: GenAIPanelDescriptionButtonProps) => { - function getMessages(): Message[] { - const dashboard = getDashboardSrv().getCurrent()!; + const messages = React.useMemo(() => getMessages(panel), [panel]); + const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.panelDescription), []); - return [ - { - content: DESCRIPTION_GENERATION_STANDARD_PROMPT, - role: Role.system, - }, - { - content: `The panel is part of a dashboard with the title: ${dashboard.title}`, - role: Role.system, - }, - { - content: `The panel is part of a dashboard with the description: ${dashboard.title}`, - role: Role.system, - }, - { - content: `Use this JSON object which defines the panel: ${JSON.stringify(panel.getSaveModel())}`, - role: Role.user, - }, - ]; - } - - return ; + return ( + + ); }; + +function getMessages(panel: PanelModel): Message[] { + const dashboard = getDashboardSrv().getCurrent()!; + + return [ + { + content: DESCRIPTION_GENERATION_STANDARD_PROMPT, + role: Role.system, + }, + { + content: `The panel is part of a dashboard with the title: ${dashboard.title}`, + role: Role.system, + }, + { + content: `The panel is part of a dashboard with the description: ${dashboard.title}`, + role: Role.system, + }, + { + content: `Use this JSON object which defines the panel: ${JSON.stringify(panel.getSaveModel())}`, + role: Role.user, + }, + ]; +} diff --git a/public/app/features/dashboard/components/GenAI/GenAIPanelTitleButton.tsx b/public/app/features/dashboard/components/GenAI/GenAIPanelTitleButton.tsx index 16f3cd22b60..f110ecb2cf8 100644 --- a/public/app/features/dashboard/components/GenAI/GenAIPanelTitleButton.tsx +++ b/public/app/features/dashboard/components/GenAI/GenAIPanelTitleButton.tsx @@ -4,6 +4,7 @@ import { getDashboardSrv } from '../../services/DashboardSrv'; import { PanelModel } from '../../state'; import { GenAIButton } from './GenAIButton'; +import { EventSource, reportGenerateAIButtonClicked } from './tracking'; import { Message, Role } from './utils'; interface GenAIPanelTitleButtonProps { @@ -17,28 +18,31 @@ const TITLE_GENERATION_STANDARD_PROMPT = 'The title should be shorter than 50 characters.'; export const GenAIPanelTitleButton = ({ onGenerate, panel }: GenAIPanelTitleButtonProps) => { - function getMessages(): Message[] { - const dashboard = getDashboardSrv().getCurrent()!; + const messages = React.useMemo(() => getMessages(panel), [panel]); + const onClick = React.useCallback(() => reportGenerateAIButtonClicked(EventSource.panelTitle), []); - return [ - { - content: TITLE_GENERATION_STANDARD_PROMPT, - role: Role.system, - }, - { - content: `The panel is part of a dashboard with the title: ${dashboard.title}`, - role: Role.system, - }, - { - content: `The panel is part of a dashboard with the description: ${dashboard.title}`, - role: Role.system, - }, - { - content: `Use this JSON object which defines the panel: ${JSON.stringify(panel.getSaveModel())}`, - role: Role.user, - }, - ]; - } - - return ; + return ; }; + +function getMessages(panel: PanelModel): Message[] { + const dashboard = getDashboardSrv().getCurrent()!; + + return [ + { + content: TITLE_GENERATION_STANDARD_PROMPT, + role: Role.system, + }, + { + content: `The panel is part of a dashboard with the title: ${dashboard.title}`, + role: Role.system, + }, + { + content: `The panel is part of a dashboard with the description: ${dashboard.title}`, + role: Role.system, + }, + { + content: `Use this JSON object which defines the panel: ${JSON.stringify(panel.getSaveModel())}`, + role: Role.user, + }, + ]; +} diff --git a/public/app/features/dashboard/components/GenAI/tracking.ts b/public/app/features/dashboard/components/GenAI/tracking.ts new file mode 100644 index 00000000000..b299c9145de --- /dev/null +++ b/public/app/features/dashboard/components/GenAI/tracking.ts @@ -0,0 +1,13 @@ +import { reportInteraction } from '@grafana/runtime'; + +export enum EventSource { + panelDescription = 'panel-description', + panelTitle = 'panel-title', + dashboardChanges = 'dashboard-changes', + dashboardTitle = 'dashboard-title', + dashboardDescription = 'dashboard-description', +} + +export function reportGenerateAIButtonClicked(src: EventSource) { + reportInteraction('dashboards_autogenerate_clicked', { src }); +} diff --git a/public/app/features/dashboard/components/GenAI/utils.ts b/public/app/features/dashboard/components/GenAI/utils.ts index d1491eb3058..918b69e5dd9 100644 --- a/public/app/features/dashboard/components/GenAI/utils.ts +++ b/public/app/features/dashboard/components/GenAI/utils.ts @@ -42,7 +42,7 @@ export const OPEN_AI_MODEL = 'gpt-4'; * * @param messages messages to send to LLM * @param onReply callback to call when LLM replies. The reply will be streamed, so it will be called for every token received. - * @param temperature what temperature to use when calling the llm. default 1. + * @param temperature what temperature to use when calling the llm. default 1. Closer to 0 means more conservative, closer to 1 means more creative. * @returns The subscription to the stream. */ export const generateTextWithLLM = async (