From 35df3ae55495cc3ca281089e1d0ea1c66cacd7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=A4ggmark?= Date: Tue, 16 Sep 2025 14:13:06 +0200 Subject: [PATCH] Chore: filter out bots from frontend logging (#111157) * Chore: filter out bots from frontend logging * chore: updates after PR feedback * chore: change solution to use beforeSend --- .../src/types/featureToggles.gen.ts | 5 ++ pkg/services/featuremgmt/registry.go | 8 +++ pkg/services/featuremgmt/toggles_gen.csv | 1 + pkg/services/featuremgmt/toggles_gen.go | 4 ++ pkg/services/featuremgmt/toggles_gen.json | 14 +++++ .../GrafanaJavascriptAgentBackend.test.ts | 10 ++++ .../GrafanaJavascriptAgentBackend.ts | 2 + .../beforeSendHandler.test.ts | 53 +++++++++++++++++++ .../beforeSendHandler.ts | 24 +++++++++ 9 files changed, 121 insertions(+) create mode 100644 public/app/core/services/echo/backends/grafana-javascript-agent/beforeSendHandler.test.ts create mode 100644 public/app/core/services/echo/backends/grafana-javascript-agent/beforeSendHandler.ts diff --git a/packages/grafana-data/src/types/featureToggles.gen.ts b/packages/grafana-data/src/types/featureToggles.gen.ts index f102e308fa0..40eb7c72c2e 100644 --- a/packages/grafana-data/src/types/featureToggles.gen.ts +++ b/packages/grafana-data/src/types/featureToggles.gen.ts @@ -1167,4 +1167,9 @@ export interface FeatureToggles { * @default false */ pluginContainers?: boolean; + /** + * Filter out bots from collecting data for Frontend Observability + * @default false + */ + filterOutBotsFromFrontendLogs?: boolean; } diff --git a/pkg/services/featuremgmt/registry.go b/pkg/services/featuremgmt/registry.go index c627b0e20a3..8e0a09f5339 100644 --- a/pkg/services/featuremgmt/registry.go +++ b/pkg/services/featuremgmt/registry.go @@ -2022,6 +2022,14 @@ var ( Expression: "false", RequiresRestart: true, }, + { + Name: "filterOutBotsFromFrontendLogs", + Description: "Filter out bots from collecting data for Frontend Observability", + Stage: FeatureStageExperimental, + FrontendOnly: true, + Owner: grafanaPluginsPlatformSquad, + Expression: "false", + }, } ) diff --git a/pkg/services/featuremgmt/toggles_gen.csv b/pkg/services/featuremgmt/toggles_gen.csv index 3ea41059b7e..12f119f9a42 100644 --- a/pkg/services/featuremgmt/toggles_gen.csv +++ b/pkg/services/featuremgmt/toggles_gen.csv @@ -259,3 +259,4 @@ azureResourcePickerUpdates,preview,@grafana/partner-datasources,false,false,true prometheusTypeMigration,experimental,@grafana/partner-datasources,false,true,false dskitBackgroundServices,experimental,@grafana/plugins-platform-backend,false,true,false pluginContainers,privatePreview,@grafana/plugins-platform-backend,false,true,false +filterOutBotsFromFrontendLogs,experimental,@grafana/plugins-platform-backend,false,false,true diff --git a/pkg/services/featuremgmt/toggles_gen.go b/pkg/services/featuremgmt/toggles_gen.go index bbfbebf2e32..1dd473a1e55 100644 --- a/pkg/services/featuremgmt/toggles_gen.go +++ b/pkg/services/featuremgmt/toggles_gen.go @@ -1046,4 +1046,8 @@ const ( // FlagPluginContainers // Enables running plugins in containers FlagPluginContainers = "pluginContainers" + + // FlagFilterOutBotsFromFrontendLogs + // Filter out bots from collecting data for Frontend Observability + FlagFilterOutBotsFromFrontendLogs = "filterOutBotsFromFrontendLogs" ) diff --git a/pkg/services/featuremgmt/toggles_gen.json b/pkg/services/featuremgmt/toggles_gen.json index a8bd59da962..10b3fbbd00f 100644 --- a/pkg/services/featuremgmt/toggles_gen.json +++ b/pkg/services/featuremgmt/toggles_gen.json @@ -1562,6 +1562,20 @@ "hideFromDocs": true } }, + { + "metadata": { + "name": "filterOutBotsFromFrontendLogs", + "resourceVersion": "1758000919535", + "creationTimestamp": "2025-09-16T05:35:19Z" + }, + "spec": { + "description": "Filter out bots from collecting data for Frontend Observability", + "stage": "experimental", + "codeowner": "@grafana/plugins-platform-backend", + "frontend": true, + "expression": "false" + } + }, { "metadata": { "name": "foldersAppPlatformAPI", diff --git a/public/app/core/services/echo/backends/grafana-javascript-agent/GrafanaJavascriptAgentBackend.test.ts b/public/app/core/services/echo/backends/grafana-javascript-agent/GrafanaJavascriptAgentBackend.test.ts index 13d18fb8977..cf6bdb9da62 100644 --- a/public/app/core/services/echo/backends/grafana-javascript-agent/GrafanaJavascriptAgentBackend.test.ts +++ b/public/app/core/services/echo/backends/grafana-javascript-agent/GrafanaJavascriptAgentBackend.test.ts @@ -4,6 +4,7 @@ import { Faro, Instrumentation } from '@grafana/faro-core'; import * as faroWebSdkModule from '@grafana/faro-web-sdk'; import { BrowserConfig, FetchTransport, SessionInstrumentation } from '@grafana/faro-web-sdk'; import { TracingInstrumentation } from '@grafana/faro-web-tracing'; +import { config } from '@grafana/runtime'; import { EchoSrvTransport } from './EchoSrvTransport'; import { @@ -46,6 +47,8 @@ describe('GrafanaJavascriptAgentEchoBackend', () => { instrumentations: mockedInstrumentations, internalLogger: mockedInternalLogger, }); + + config.featureToggles.filterOutBotsFromFrontendLogs = false; }); afterEach(() => { @@ -154,6 +157,13 @@ describe('GrafanaJavascriptAgentEchoBackend', () => { expect(initializeFaroMock.mock.calls[1][0].instrumentations?.[1]).toBeInstanceOf(SessionInstrumentation); }); + it('should use a beforeSend handler', () => { + new GrafanaJavascriptAgentBackend(options); + + expect(initializeFaroMock).toHaveBeenCalledTimes(1); + expect(initializeFaroMock.mock.calls[0][0].beforeSend).toBeDefined(); + }); + //@FIXME - make integration test work // it('integration test with EchoSrv and GrafanaJavascriptAgent', async () => { diff --git a/public/app/core/services/echo/backends/grafana-javascript-agent/GrafanaJavascriptAgentBackend.ts b/public/app/core/services/echo/backends/grafana-javascript-agent/GrafanaJavascriptAgentBackend.ts index f6988c93d1e..e2ec294ff23 100644 --- a/public/app/core/services/echo/backends/grafana-javascript-agent/GrafanaJavascriptAgentBackend.ts +++ b/public/app/core/services/echo/backends/grafana-javascript-agent/GrafanaJavascriptAgentBackend.ts @@ -16,6 +16,7 @@ import { TracingInstrumentation } from '@grafana/faro-web-tracing'; import { EchoBackend, EchoEvent, EchoEventType } from '@grafana/runtime'; import { EchoSrvTransport } from './EchoSrvTransport'; +import { beforeSendHandler } from './beforeSendHandler'; import { GrafanaJavascriptAgentEchoEvent, User } from './types'; function isCrossOriginIframe() { @@ -118,6 +119,7 @@ export class GrafanaJavascriptAgentBackend sendTimeout: 1000, }, internalLoggerLevel: options.internalLoggerLevel || defaultInternalLoggerLevel, + beforeSend: beforeSendHandler, }; this.faroInstance = initializeFaro(grafanaJavaScriptAgentOptions); diff --git a/public/app/core/services/echo/backends/grafana-javascript-agent/beforeSendHandler.test.ts b/public/app/core/services/echo/backends/grafana-javascript-agent/beforeSendHandler.test.ts new file mode 100644 index 00000000000..e78e3a50fc5 --- /dev/null +++ b/public/app/core/services/echo/backends/grafana-javascript-agent/beforeSendHandler.test.ts @@ -0,0 +1,53 @@ +import { TransportItem, TransportItemType } from '@grafana/faro-core'; +import { config } from '@grafana/runtime'; + +import { beforeSendHandler } from './beforeSendHandler'; + +const getTransportationItem = (userAgent: string | undefined): TransportItem => ({ + meta: { browser: { userAgent } }, + payload: {}, + type: TransportItemType.LOG, +}); + +describe('beforeSendHandler', () => { + beforeEach(() => { + config.featureToggles.filterOutBotsFromFrontendLogs = false; + }); + + it('should return item when feature toggle is disabled', () => { + const botUserAgent = 'Googlebot/2.1 (+http://www.google.com/bot.html)'; + const item = getTransportationItem(botUserAgent); + expect(beforeSendHandler(item)).toBe(item); + }); + + it('should return item for regular user agents', () => { + config.featureToggles.filterOutBotsFromFrontendLogs = true; + const regularUserAgent = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36'; + const item = getTransportationItem(regularUserAgent); + expect(beforeSendHandler(item)).toBe(item); + }); + + it.each(['', undefined])('should return item when user agent is %s', (userAgent) => { + config.featureToggles.filterOutBotsFromFrontendLogs = true; + const item = getTransportationItem(userAgent); + expect(beforeSendHandler(item)).toBe(item); + }); + + it.each([ + 'Googlebot/2.1 (+http://www.google.com/bot.html)', + 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)', + 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)', + 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)', + 'Twitterbot/1.0', + 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)', + 'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)', + 'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)', + 'Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)', + 'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)', + ])('should return 0 for bot user agent: %s', (userAgent) => { + config.featureToggles.filterOutBotsFromFrontendLogs = true; + const item = getTransportationItem(userAgent); + expect(beforeSendHandler(item)).toBe(null); + }); +}); diff --git a/public/app/core/services/echo/backends/grafana-javascript-agent/beforeSendHandler.ts b/public/app/core/services/echo/backends/grafana-javascript-agent/beforeSendHandler.ts new file mode 100644 index 00000000000..95e65acc826 --- /dev/null +++ b/public/app/core/services/echo/backends/grafana-javascript-agent/beforeSendHandler.ts @@ -0,0 +1,24 @@ +import { TransportItem } from '@grafana/faro-core'; +import { config } from '@grafana/runtime'; + +// as listed in https://grafana.com/docs/grafana-cloud/monitor-applications/frontend-observability/instrument/filter-bots/#filter-out-bots-from-collecting-data-for-frontend-observability +const bots = + '(googlebot|Googlebot-Mobile|Googlebot-Image|Google favicon|Mediapartners-Google|bingbot|slurp|Commons-HttpClient|Python-urllib|libwww|httpunit|nutch|phpcrawl|msnbot|jyxobot|FAST-WebCrawler|FAST Enterprise Crawler|biglotron|teoma|convera|seekbot|gigablast|exabot|ngbot|ia_archiver|GingerCrawler|webmon |httrack|webcrawler|grub.org|UsineNouvelleCrawler|antibot|netresearchserver|speedy|fluffy|bibnum.bnf|findlink|msrbot|panscient|yacybot|AISearchBot|IOI|ips-agent|tagoobot|MJ12bot|dotbot|woriobot|yanga|buzzbot|mlbot|yandexbot|purebot|Linguee Bot|Voyager|CyberPatrol|voilabot|baiduspider|citeseerxbot|spbot|twengabot|postrank|turnitinbot|scribdbot|page2rss|sitebot|linkdex|Adidxbot|blekkobot|ezooms|Mail.RU_Bot|discobot|heritrix|findthatfile|europarchive.org|NerdByNature.Bot|sistrix crawler|ahrefsbot|Aboundex|domaincrawler|wbsearchbot|summify|ccbot|edisterbot|seznambot|ec2linkfinder|gslfbot|aihitbot|intelium_bot|facebookexternalhit|yeti|RetrevoPageAnalyzer|lb-spider|sogou|lssbot|careerbot|wotbox|wocbot|ichiro|DuckDuckBot|lssrocketcrawler|drupact|webcompanycrawler|acoonbot|openindexspider|gnam gnam spider|web-archive-net.com.bot|backlinkcrawler|coccoc|integromedb|content crawler spider|toplistbot|seokicks-robot|it2media-domain-crawler|ip-web-crawler.com|siteexplorer.info|elisabot|proximic|changedetection|blexbot|arabot|WeSEE:Search|niki-bot|CrystalSemanticsBot|rogerbot|360Spider|psbot|InterfaxScanBot|Lipperhey SEO Service|CC Metadata Scaper|g00g1e.net|GrapeshotCrawler|urlappendbot|brainobot|fr-crawler|binlar|SimpleCrawler|Livelapbot|Twitterbot|cXensebot|smtbot|bnf.fr_bot|A6-Indexer|ADmantX|Facebot|OrangeBot|memorybot|AdvBot|MegaIndex|SemanticScholarBot|ltx71|nerdybot|xovibot|BUbiNG|Qwantify|archive.org_bot|Applebot|TweetmemeBot|crawler4j|findxbot|SemrushBot|yoozBot|lipperhey|y!j-asr|Domain Re-Animator Bot|AddThis)'; + +const botsRegex = new RegExp(bots, 'i'); + +export function beforeSendHandler(item: TransportItem): TransportItem | null { + if (!config.featureToggles.filterOutBotsFromFrontendLogs) { + return item; + } + + if (!item.meta.browser?.userAgent) { + return item; + } + + if (botsRegex.test(item.meta.browser.userAgent)) { + return null; + } + + return item; +}