Chore: replace feature toggle filterOutBotsFromFrontendLogs with config (#112355)

* Chore: replace feature toggle with config

* chore: small fixes

* chore: updates after pr feedback

* chore: fixes writers toolkit error
This commit is contained in:
Hugo Häggmark
2025-10-16 12:03:07 +00:00
committed by GitHub
parent ffee922fae
commit a30a71905e
16 changed files with 125 additions and 92 deletions
+2
View File
@@ -1234,6 +1234,8 @@ log_endpoint_requests_per_second_limit = 3
# Max requests accepted per short interval of time for Grafana backend log ingestion endpoint (/log)
log_endpoint_burst_limit = 15
# Enables the bot filter for the Grafana Faro JavaScript agent integration. Default is `false`. When enabled, it will filter out requests from known bots and crawlers.
bot_filter_enabled = false
#################################### Usage Quotas ########################
[quota]
+3
View File
@@ -1206,6 +1206,9 @@
# See https://grafana.com/docs/grafana-cloud/monitor-applications/frontend-observability/instrument/web-vitals/#web-vitals-attribution-data
;web_vitals_attribution_enabled = true
# Enables the bot filter for the Grafana Faro JavaScript agent integration. Default is `false`. When enabled, it will filter out requests from known bots and crawlers.
;bot_filter_enabled = false
#################################### Usage Quotas ########################
[quota]
; enabled = false
@@ -1679,6 +1679,10 @@ Requests per second limit enforced per an extended period, for Grafana backend l
Maximum requests accepted per short interval of time for Grafana backend log ingestion endpoint, `/log-grafana-javascript-agent`. Default is `15`.
#### `bot_filter_enabled`
Enables the bot filter for the Grafana Faro JavaScript agent integration. Default is `false`. When enabled, it will filter out requests from known bots and crawlers.
<hr>
### `[quota]`
@@ -1200,11 +1200,6 @@ export interface FeatureToggles {
*/
tempoSearchBackendMigration?: boolean;
/**
* Filter out bots from collecting data for Frontend Observability
* @default false
*/
filterOutBotsFromFrontendLogs?: boolean;
/**
* Prioritize loading plugins from the CDN before other sources
* @default false
*/
+1
View File
@@ -161,6 +161,7 @@ export class GrafanaBootConfig {
tracingInstrumentalizationEnabled: false,
webVitalsAttribution: false,
internalLoggerLevel: 0,
botFilterEnabled: false,
};
pluginCatalogURL = 'https://grafana.com/grafana/plugins/';
pluginAdminEnabled = true;
-8
View File
@@ -2080,14 +2080,6 @@ var (
Expression: "false",
RequiresRestart: true,
},
{
Name: "filterOutBotsFromFrontendLogs",
Description: "Filter out bots from collecting data for Frontend Observability",
Stage: FeatureStageExperimental,
FrontendOnly: true,
Owner: grafanaPluginsPlatformSquad,
Expression: "false",
},
{
Name: "cdnPluginsLoadFirst",
Description: "Prioritize loading plugins from the CDN before other sources",
-1
View File
@@ -267,6 +267,5 @@ azureResourcePickerUpdates,preview,@grafana/partner-datasources,false,false,true
prometheusTypeMigration,experimental,@grafana/partner-datasources,false,true,false
pluginContainers,privatePreview,@grafana/plugins-platform-backend,false,true,false
tempoSearchBackendMigration,GA,@grafana/oss-big-tent,false,true,false
filterOutBotsFromFrontendLogs,experimental,@grafana/plugins-platform-backend,false,false,true
cdnPluginsLoadFirst,experimental,@grafana/plugins-platform-backend,false,false,false
cdnPluginsUrls,experimental,@grafana/plugins-platform-backend,false,false,false
1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
267 prometheusTypeMigration experimental @grafana/partner-datasources false true false
268 pluginContainers privatePreview @grafana/plugins-platform-backend false true false
269 tempoSearchBackendMigration GA @grafana/oss-big-tent false true false
filterOutBotsFromFrontendLogs experimental @grafana/plugins-platform-backend false false true
270 cdnPluginsLoadFirst experimental @grafana/plugins-platform-backend false false false
271 cdnPluginsUrls experimental @grafana/plugins-platform-backend false false false
-4
View File
@@ -1078,10 +1078,6 @@ const (
// Run search queries through the tempo backend
FlagTempoSearchBackendMigration = "tempoSearchBackendMigration"
// FlagFilterOutBotsFromFrontendLogs
// Filter out bots from collecting data for Frontend Observability
FlagFilterOutBotsFromFrontendLogs = "filterOutBotsFromFrontendLogs"
// FlagCdnPluginsLoadFirst
// Prioritize loading plugins from the CDN before other sources
FlagCdnPluginsLoadFirst = "cdnPluginsLoadFirst"
+2 -1
View File
@@ -1692,7 +1692,8 @@
"metadata": {
"name": "filterOutBotsFromFrontendLogs",
"resourceVersion": "1758000919535",
"creationTimestamp": "2025-09-16T05:35:19Z"
"creationTimestamp": "2025-09-16T05:35:19Z",
"deletionTimestamp": "2025-10-13T11:09:22Z"
},
"spec": {
"description": "Filter out bots from collecting data for Frontend Observability",
@@ -14,6 +14,7 @@ type GrafanaJavascriptAgent struct {
CSPInstrumentalizationEnabled bool `json:"cspInstrumentalizationEnabled"`
TracingInstrumentalizationEnabled bool `json:"tracingInstrumentalizationEnabled"`
WebVitalsAttributionEnabled bool `json:"webVitalsAttributionEnabled"`
BotFilterEnabled bool `json:"botFilterEnabled"`
}
func (cfg *Cfg) readGrafanaJavascriptAgentConfig() {
@@ -32,5 +33,6 @@ func (cfg *Cfg) readGrafanaJavascriptAgentConfig() {
CSPInstrumentalizationEnabled: raw.Key("instrumentations_csp_enabled").MustBool(true),
TracingInstrumentalizationEnabled: raw.Key("instrumentations_tracing_enabled").MustBool(true),
WebVitalsAttributionEnabled: raw.Key("web_vitals_attribution_enabled").MustBool(true),
BotFilterEnabled: raw.Key("bot_filter_enabled").MustBool(false),
}
}
@@ -12,14 +12,10 @@ import {
ViewInstrumentation,
} from '@grafana/faro-web-sdk';
import { TracingInstrumentation } from '@grafana/faro-web-tracing';
import { config } from '@grafana/runtime';
import { EchoSrvTransport } from './EchoSrvTransport';
import {
GrafanaJavascriptAgentBackend,
GrafanaJavascriptAgentBackendOptions,
TRACKING_URLS,
} from './GrafanaJavascriptAgentBackend';
import { GrafanaJavascriptAgentBackend, TRACKING_URLS } from './GrafanaJavascriptAgentBackend';
import { GrafanaJavascriptAgentBackendOptions } from './types';
describe('GrafanaJavascriptAgentEchoBackend', () => {
let mockedSetUser: jest.Mock;
@@ -55,8 +51,6 @@ describe('GrafanaJavascriptAgentEchoBackend', () => {
instrumentations: mockedInstrumentations,
internalLogger: mockedInternalLogger,
});
config.featureToggles.filterOutBotsFromFrontendLogs = false;
});
afterEach(() => {
@@ -90,6 +84,7 @@ describe('GrafanaJavascriptAgentEchoBackend', () => {
buildInfo: buildInfo,
userIdentifier: 'abc123',
ignoreUrls: [],
botFilterEnabled: false,
};
it('will set up FetchTransport if customEndpoint is provided', () => {
@@ -1,5 +1,5 @@
import { BuildInfo, escapeRegex } from '@grafana/data';
import { BaseTransport, defaultInternalLoggerLevel, InternalLoggerLevel } from '@grafana/faro-core';
import { escapeRegex } from '@grafana/data';
import { BaseTransport, defaultInternalLoggerLevel } from '@grafana/faro-core';
import {
initializeFaro,
BrowserConfig,
@@ -12,7 +12,7 @@ import { EchoBackend, EchoEvent, EchoEventType } from '@grafana/runtime';
import { EchoSrvTransport } from './EchoSrvTransport';
import { beforeSendHandler } from './beforeSendHandler';
import { GrafanaJavascriptAgentEchoEvent } from './types';
import { GrafanaJavascriptAgentBackendOptions, GrafanaJavascriptAgentEchoEvent } from './types';
function isCrossOriginIframe() {
try {
@@ -22,22 +22,6 @@ function isCrossOriginIframe() {
}
}
export interface GrafanaJavascriptAgentBackendOptions {
apiKey?: string;
customEndpoint?: string;
internalLoggerLevel?: InternalLoggerLevel;
webVitalsAttribution: boolean;
consoleInstrumentalizationEnabled: boolean;
performanceInstrumentalizationEnabled: boolean;
cspInstrumentalizationEnabled: boolean;
tracingInstrumentalizationEnabled: boolean;
buildInfo: BuildInfo;
userIdentifier: string;
ignoreUrls: RegExp[];
}
export const TRACKING_URLS = [
/\.(google-analytics|googletagmanager)\.com/,
/frontend-metrics/,
@@ -107,7 +91,7 @@ export class GrafanaJavascriptAgentBackend
batching: {
sendTimeout: 1000,
},
beforeSend: beforeSendHandler,
beforeSend: (item) => beforeSendHandler(options.botFilterEnabled, item),
internalLoggerLevel: options.internalLoggerLevel ?? defaultInternalLoggerLevel,
};
@@ -1,5 +1,4 @@
import { TransportItem, TransportItemType } from '@grafana/faro-core';
import { config } from '@grafana/runtime';
import { beforeSendHandler } from './beforeSendHandler';
@@ -11,43 +10,53 @@ const getTransportationItem = (userAgent: string | undefined): TransportItem =>
describe('beforeSendHandler', () => {
beforeEach(() => {
config.featureToggles.filterOutBotsFromFrontendLogs = false;
jest.spyOn(console, 'warn').mockImplementation(() => {});
});
it('should return item when feature toggle is disabled', () => {
const botUserAgent = 'Googlebot/2.1 (+http://www.google.com/bot.html)';
const item = getTransportationItem(botUserAgent);
expect(beforeSendHandler(item)).toBe(item);
describe('when botFilterEnabled is false', () => {
it('should return item', () => {
const botUserAgent = 'Googlebot/2.1 (+http://www.google.com/bot.html)';
const item = getTransportationItem(botUserAgent);
expect(beforeSendHandler(false, item)).toBe(item);
});
});
it('should return item for regular user agents', () => {
config.featureToggles.filterOutBotsFromFrontendLogs = true;
const regularUserAgent =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36';
const item = getTransportationItem(regularUserAgent);
expect(beforeSendHandler(item)).toBe(item);
});
describe('when botFilterEnabled is true', () => {
const userUserAgents = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:120.0) Gecko/20100101 Firefox/120.0',
];
const invalidUserAgents = ['', ' ', undefined, null, 0, 1, {}, [], () => {}];
const maliciousUserAgents = userUserAgents.map((ua) => ua + 'a'.repeat(600));
it.each(['', undefined])('should return item when user agent is %s', (userAgent) => {
config.featureToggles.filterOutBotsFromFrontendLogs = true;
const item = getTransportationItem(userAgent);
expect(beforeSendHandler(item)).toBe(item);
});
it.each(userUserAgents)('should return item for bot user agent: %s', (userAgent) => {
const item = getTransportationItem(userAgent);
expect(beforeSendHandler(true, item)).toBe(item);
});
it.each([
'Googlebot/2.1 (+http://www.google.com/bot.html)',
'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
'Twitterbot/1.0',
'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)',
'Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)',
'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)',
])('should return 0 for bot user agent: %s', (userAgent) => {
config.featureToggles.filterOutBotsFromFrontendLogs = true;
const item = getTransportationItem(userAgent);
expect(beforeSendHandler(item)).toBe(null);
it.each([
...invalidUserAgents,
...maliciousUserAgents,
'Googlebot/2.1 (+http://www.google.com/bot.html)',
'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)',
'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
'Twitterbot/1.0',
'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)',
'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)',
'Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)',
'Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)',
'Mozilla/5.0 (compatible; NOTgooglebot/2.1)',
'Mozilla/5.0 (compatible; googlebotbypass/2.1)',
'Mozilla/5.0 (compatible; notbingbot/2.0; +http://www.bing.com/notbingbot.htm)',
])('should return null for bot user agent: %s', (userAgent) => {
const item = getTransportationItem(userAgent as string);
expect(beforeSendHandler(true, item)).toBe(null);
});
});
});
@@ -1,24 +1,55 @@
import { TransportItem } from '@grafana/faro-core';
import { config } from '@grafana/runtime';
// as listed in https://grafana.com/docs/grafana-cloud/monitor-applications/frontend-observability/instrument/filter-bots/#filter-out-bots-from-collecting-data-for-frontend-observability
const bots =
'(googlebot|Googlebot-Mobile|Googlebot-Image|Google favicon|Mediapartners-Google|bingbot|slurp|Commons-HttpClient|Python-urllib|libwww|httpunit|nutch|phpcrawl|msnbot|jyxobot|FAST-WebCrawler|FAST Enterprise Crawler|biglotron|teoma|convera|seekbot|gigablast|exabot|ngbot|ia_archiver|GingerCrawler|webmon |httrack|webcrawler|grub.org|UsineNouvelleCrawler|antibot|netresearchserver|speedy|fluffy|bibnum.bnf|findlink|msrbot|panscient|yacybot|AISearchBot|IOI|ips-agent|tagoobot|MJ12bot|dotbot|woriobot|yanga|buzzbot|mlbot|yandexbot|purebot|Linguee Bot|Voyager|CyberPatrol|voilabot|baiduspider|citeseerxbot|spbot|twengabot|postrank|turnitinbot|scribdbot|page2rss|sitebot|linkdex|Adidxbot|blekkobot|ezooms|Mail.RU_Bot|discobot|heritrix|findthatfile|europarchive.org|NerdByNature.Bot|sistrix crawler|ahrefsbot|Aboundex|domaincrawler|wbsearchbot|summify|ccbot|edisterbot|seznambot|ec2linkfinder|gslfbot|aihitbot|intelium_bot|facebookexternalhit|yeti|RetrevoPageAnalyzer|lb-spider|sogou|lssbot|careerbot|wotbox|wocbot|ichiro|DuckDuckBot|lssrocketcrawler|drupact|webcompanycrawler|acoonbot|openindexspider|gnam gnam spider|web-archive-net.com.bot|backlinkcrawler|coccoc|integromedb|content crawler spider|toplistbot|seokicks-robot|it2media-domain-crawler|ip-web-crawler.com|siteexplorer.info|elisabot|proximic|changedetection|blexbot|arabot|WeSEE:Search|niki-bot|CrystalSemanticsBot|rogerbot|360Spider|psbot|InterfaxScanBot|Lipperhey SEO Service|CC Metadata Scaper|g00g1e.net|GrapeshotCrawler|urlappendbot|brainobot|fr-crawler|binlar|SimpleCrawler|Livelapbot|Twitterbot|cXensebot|smtbot|bnf.fr_bot|A6-Indexer|ADmantX|Facebot|OrangeBot|memorybot|AdvBot|MegaIndex|SemanticScholarBot|ltx71|nerdybot|xovibot|BUbiNG|Qwantify|archive.org_bot|Applebot|TweetmemeBot|crawler4j|findxbot|SemrushBot|yoozBot|lipperhey|y!j-asr|Domain Re-Animator Bot|AddThis)';
'(googlebot|googlebot-mobile|googlebot-image|google favicon|mediapartners-google|' +
'bingbot|slurp|commons-httpclient|python-urllib|libwww|httpunit|nutch|phpcrawl|' +
'msnbot|jyxobot|fast-webcrawler|fast enterprise crawler|biglotron|teoma|convera|' +
'seekbot|gigablast|exabot|ngbot|ia_archiver|gingercrawler|webmon |httrack|' +
'webcrawler|grub.org|usinenouvellecrawler|antibot|netresearchserver|speedy|fluffy|' +
'bibnum.bnf|findlink|msrbot|panscient|yacybot|aisearchbot|ioi|ips-agent|tagoobot|' +
'mj12bot|dotbot|woriobot|yanga|buzzbot|mlbot|yandexbot|purebot|linguee bot|voyager|' +
'cyberpatrol|voilabot|baiduspider|citeseerxbot|spbot|twengabot|postrank|turnitinbot|' +
'scribdbot|page2rss|sitebot|linkdex|adidxbot|blekkobot|ezooms|mail.ru_bot|discobot|' +
'heritrix|findthatfile|europarchive.org|nerdbynature.bot|sistrix crawler|ahrefsbot|' +
'aboundex|domaincrawler|wbsearchbot|summify|ccbot|edisterbot|seznambot|ec2linkfinder|' +
'gslfbot|aihitbot|intelium_bot|facebookexternalhit|yeti|retrevopageanalyzer|lb-spider|' +
'sogou|lssbot|careerbot|wotbox|wocbot|ichiro|duckduckbot|lssrocketcrawler|drupact|' +
'webcompanycrawler|acoonbot|openindexspider|gnam gnam spider|web-archive-net.com.bot|' +
'backlinkcrawler|coccoc|integromedb|content crawler spider|toplistbot|seokicks-robot|' +
'it2media-domain-crawler|ip-web-crawler.com|siteexplorer.info|elisabot|proximic|' +
'changedetection|blexbot|arabot|wesee:search|niki-bot|crystalsemanticsbot|rogerbot|' +
'360spider|psbot|interfaxscanbot|lipperhey seo service|cc metadata scraper|g00g1e.net|' +
'grapeshotcrawler|urlappendbot|brainobot|fr-crawler|binlar|simplecrawler|livelapbot|' +
'twitterbot|cxensebot|smtbot|bnf.fr_bot|a6-indexer|admantx|facebot|orangebot|' +
'memorybot|advbot|megaindex|semanticscholarbot|ltx71|nerdybot|xovibot|bubing|' +
'qwantify|archive.org_bot|applebot|tweetmemebot|crawler4j|findxbot|semrushbot|' +
'yoozbot|lipperhey|y!j-asr|domain re-animator bot|addthis)';
const botsRegex = new RegExp(bots, 'i');
const botsRegex = new RegExp(bots);
export function beforeSendHandler(item: TransportItem): TransportItem | null {
if (!config.featureToggles.filterOutBotsFromFrontendLogs) {
export function beforeSendHandler(botFilterEnabled: boolean, item: TransportItem): TransportItem | null {
if (!botFilterEnabled) {
return item;
}
if (!item.meta.browser?.userAgent) {
return item;
}
if (botsRegex.test(item.meta.browser.userAgent)) {
if (typeof item.meta.browser?.userAgent !== 'string') {
return null;
}
return item;
const userAgent = item.meta.browser?.userAgent?.trim().toLowerCase();
if (!userAgent) {
return null;
}
if (userAgent.length > 512) {
return null;
}
try {
const isBot = botsRegex.test(userAgent);
return isBot ? null : item;
} catch (error) {
return null;
}
}
@@ -1,4 +1,5 @@
import { CurrentUserDTO } from '@grafana/data';
import { BuildInfo, CurrentUserDTO } from '@grafana/data';
import { InternalLoggerLevel } from '@grafana/faro-web-sdk';
import { EchoEvent, EchoEventType } from '@grafana/runtime';
export interface BaseTransport {
@@ -11,3 +12,20 @@ export interface User extends Pick<CurrentUserDTO, 'email'> {
id: string;
orgId?: number;
}
export interface GrafanaJavascriptAgentBackendOptions {
apiKey?: string;
customEndpoint?: string;
internalLoggerLevel?: InternalLoggerLevel;
webVitalsAttribution: boolean;
consoleInstrumentalizationEnabled: boolean;
performanceInstrumentalizationEnabled: boolean;
cspInstrumentalizationEnabled: boolean;
tracingInstrumentalizationEnabled: boolean;
buildInfo: BuildInfo;
userIdentifier: string;
ignoreUrls: RegExp[];
botFilterEnabled: boolean;
}
+1
View File
@@ -109,6 +109,7 @@ async function initFaroBackend() {
tracingInstrumentalizationEnabled: config.grafanaJavascriptAgent.tracingInstrumentalizationEnabled,
webVitalsAttribution: config.grafanaJavascriptAgent.webVitalsAttribution,
internalLoggerLevel: config.grafanaJavascriptAgent.internalLoggerLevel,
botFilterEnabled: config.grafanaJavascriptAgent.botFilterEnabled,
})
);
}