diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 792354e405d..d6e9141838e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -737,10 +737,6 @@ embed.go @grafana/grafana-as-code /public/app/plugins/*gen.go @grafana/grafana-as-code /cue.mod/ @grafana/grafana-as-code -# Rendered OpenAPI from app platform -# Eventually each file owned by the right team, OR a structure with the rendered value under /apis/{group}/openapi -/openapi/ @grafana/grafana-app-platform-squad - # GitHub Workflows and Templates /.github/CODEOWNERS @tolzhabayev /.github/ISSUE_TEMPLATE/ @torkelo @sympatheticmoose diff --git a/CHANGELOG.md b/CHANGELOG.md index c1c7060ba7a..927936a27d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,171 @@ + + +# 11.5.0 (2025-01-28) + +### Features and enhancements + +- ** CloudMigration:** Create authapi service [#96581](https://github.com/grafana/grafana/pull/96581), [@leandro-deveikis](https://github.com/leandro-deveikis) +- **Alerting:** Add new button for exporting new alert rule in HCL format [#96785](https://github.com/grafana/grafana/pull/96785), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Add option to show inactive alerts in alert list panel [#96888](https://github.com/grafana/grafana/pull/96888), [@bradleypettit](https://github.com/bradleypettit) +- **Alerting:** Add state_periodic_save_batch_size config option [#98019](https://github.com/grafana/grafana/pull/98019), [@alexander-akhmetov](https://github.com/alexander-akhmetov) +- **Alerting:** Change default for max_attempts to 3. [#97461](https://github.com/grafana/grafana/pull/97461), [@stevesg](https://github.com/stevesg) +- **Alerting:** Consume k8s API for notification policies tree [#96147](https://github.com/grafana/grafana/pull/96147), [@konrad147](https://github.com/konrad147) +- **Alerting:** Enable flag alertingApiServer by default [#98282](https://github.com/grafana/grafana/pull/98282), [@yuri-tceretian](https://github.com/yuri-tceretian) +- **Alerting:** Explore button in Insights view [#96496](https://github.com/grafana/grafana/pull/96496), [@ppcano](https://github.com/ppcano) +- **Alerting:** Improve performance ash page [#97619](https://github.com/grafana/grafana/pull/97619), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Make alert rule policies preview use k8s API [#97070](https://github.com/grafana/grafana/pull/97070), [@tomratcliffe](https://github.com/tomratcliffe) +- **Alerting:** Return default builtin templates in k8s templategroup API and UI [#96330](https://github.com/grafana/grafana/pull/96330), [@JacobsonMT](https://github.com/JacobsonMT) +- **Alerting:** Simplify notification step [#96430](https://github.com/grafana/grafana/pull/96430), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Update state manager to take image only once per rule evaluation [#98289](https://github.com/grafana/grafana/pull/98289), [@yuri-tceretian](https://github.com/yuri-tceretian) +- **Analytics Views:** Deprecate :dashboardID endpoints in favor of uid/:dashboardUID (Enterprise) +- **Analytics:** Summaries: Deprecate dashboard_id endpoints in favor of dashboard_uid (Enterprise) +- **Announcement Banners:** Enable feature for all cloud tiers (Enterprise) +- **Announcement banner:** Remove feature toggle [#98782](https://github.com/grafana/grafana/pull/98782), [@Clarity-89](https://github.com/Clarity-89) +- **Announcement banner:** Remove feature toggle (Enterprise) +- **Announcement banner:** Sort by last updated (Enterprise) +- **Auth:** Return error when retries have been exhausted for OAuth token refresh [#98034](https://github.com/grafana/grafana/pull/98034), [@mgyongyosi](https://github.com/mgyongyosi) +- **Azure Monitor:** Add a feature flag to toggle user auth for Azure Monitor only [#96858](https://github.com/grafana/grafana/pull/96858), [@adamyeats](https://github.com/adamyeats) +- **Azure:** Improve Azure Prometheus exemplars UI/UX [#97198](https://github.com/grafana/grafana/pull/97198), [@aangelisc](https://github.com/aangelisc) +- **Azure:** Unify credentials in frontend for Prometheus [#96568](https://github.com/grafana/grafana/pull/96568), [@yjsong11](https://github.com/yjsong11) +- **Chore:** Bump Go to 1.23.4 [#98853](https://github.com/grafana/grafana/pull/98853), [@Proximyst](https://github.com/Proximyst) +- **Chore:** Bump Go to 1.23.4 (Enterprise) +- **Chore:** Remove experimental Storage UI [#96887](https://github.com/grafana/grafana/pull/96887), [@ryantxu](https://github.com/ryantxu) +- **Chore:** Update to node 22 [#97779](https://github.com/grafana/grafana/pull/97779), [@ashharrison90](https://github.com/ashharrison90) +- **CloudMigrations:** Enable feature toggle by default in 11.5 [#98686](https://github.com/grafana/grafana/pull/98686), [@mmandrus](https://github.com/mmandrus) +- **CloudMigrations:** Introduce RBAC role for migration assistant [#98588](https://github.com/grafana/grafana/pull/98588), [@macabu](https://github.com/macabu) +- **CloudWatch:** Add OpenSearch PPL and SQL support in Logs Insights [#97508](https://github.com/grafana/grafana/pull/97508), [@idastambuk](https://github.com/idastambuk) +- **CloudWatch:** Batch different time ranges separately [#98230](https://github.com/grafana/grafana/pull/98230), [@iwysiu](https://github.com/iwysiu) +- **Cloudwatch:** Accept empty string for logstimeout and mark errors downstream [#96947](https://github.com/grafana/grafana/pull/96947), [@iwysiu](https://github.com/iwysiu) +- **Cloudwatch:** Update grafana-aws-sdk for AWS/AmplifyHosting metrics [#97799](https://github.com/grafana/grafana/pull/97799), [@iwysiu](https://github.com/iwysiu) +- **Dashboard Scene:** Shows usages in variables list [#96000](https://github.com/grafana/grafana/pull/96000), [@harisrozajac](https://github.com/harisrozajac) +- **Dashboards:** Add option to specify explicit percent change text size for stat panels [#96952](https://github.com/grafana/grafana/pull/96952), [@XZCendence](https://github.com/XZCendence) +- **Dashboards:** Allow DashboardDS subqueries in MixedDS [#97116](https://github.com/grafana/grafana/pull/97116), [@mdvictor](https://github.com/mdvictor) +- **Dashboards:** Update docs of the `overwrite` param in Save Dashboard API Call [#97011](https://github.com/grafana/grafana/pull/97011), [@ArturWierzbicki](https://github.com/ArturWierzbicki) +- **Datasources:** Add toggle to control default behaviour of 'Manage alerts via Alerts UI' toggle [#98441](https://github.com/grafana/grafana/pull/98441), [@macabu](https://github.com/macabu) +- **Datasources:** Allow clearing trace to logs, metrics and profiles datasource pickers [#96554](https://github.com/grafana/grafana/pull/96554), [@adrapereira](https://github.com/adrapereira) +- **Docker:** Don't use legacy ENV syntax [#93218](https://github.com/grafana/grafana/pull/93218), [@simPod](https://github.com/simPod) +- **Elasticsearch:** Health endpoint should handle http errors [#96803](https://github.com/grafana/grafana/pull/96803), [@iwysiu](https://github.com/iwysiu) +- **Elasticsearch:** Use \_field_caps instead of \_mapping to get fields [#97607](https://github.com/grafana/grafana/pull/97607), [@iwysiu](https://github.com/iwysiu) +- **Explore Profiles:** Preinstall for onprem Grafana instances [#97775](https://github.com/grafana/grafana/pull/97775), [@ifrost](https://github.com/ifrost) +- **Explore metrics:** Consolidate filters with the OTel experience [#98371](https://github.com/grafana/grafana/pull/98371), [@bohandley](https://github.com/bohandley) +- **Explore:** Show links to queryless apps [#96625](https://github.com/grafana/grafana/pull/96625), [@ifrost](https://github.com/ifrost) +- **Expressions:** Add notification for Strict Mode behavior in Reduce component [#97224](https://github.com/grafana/grafana/pull/97224), [@shubhankarunhale](https://github.com/shubhankarunhale) +- **Faro:** Improve performance of TRACKING_URLS regex [#98022](https://github.com/grafana/grafana/pull/98022), [@kpelelis](https://github.com/kpelelis) +- **FeatureToggles:** Make newFiltersUI feature toggle generally available [#97460](https://github.com/grafana/grafana/pull/97460), [@Sergej-Vlasov](https://github.com/Sergej-Vlasov) +- **Features:** Remove cloudwatchMetricInsightsCrossAccount feature toggle [#98826](https://github.com/grafana/grafana/pull/98826), [@idastambuk](https://github.com/idastambuk) +- **Frontend Sandbox:** Add switch to toggle plugins frontend sandbox via catalog UI (Enterprise) +- **Graphite:** Set `maxDataPoints` based on user value in alerting [#97178](https://github.com/grafana/grafana/pull/97178), [@aangelisc](https://github.com/aangelisc) +- **Licensing:** Tidy up license token database code (Enterprise) +- **LoginAttempt:** Add setting to control max number of attempts before user login gets locked [#97091](https://github.com/grafana/grafana/pull/97091), [@kalleep](https://github.com/kalleep) +- **Logs Panel:** Add infinite scrolling support for Dashboards and Apps [#97095](https://github.com/grafana/grafana/pull/97095), [@matyax](https://github.com/matyax) +- **Logs Panel:** Allow text selection without changing Log Details state [#96995](https://github.com/grafana/grafana/pull/96995), [@matyax](https://github.com/matyax) +- **Logs Panel:** Limit displayed characters to MAX_CHARACTERS [#96997](https://github.com/grafana/grafana/pull/96997), [@matyax](https://github.com/matyax) +- **Logs:** Added option to show the log line body when displayed fields are used [#97209](https://github.com/grafana/grafana/pull/97209), [@matyax](https://github.com/matyax) +- **Logs:** Added support to disable and re-enable the popover menu [#98254](https://github.com/grafana/grafana/pull/98254), [@matyax](https://github.com/matyax) +- **Logs:** Allow scroll to reach the bottom of the log list before loading more [#96668](https://github.com/grafana/grafana/pull/96668), [@matyax](https://github.com/matyax) +- **Loki:** Added support for disabled operations in Query Builder [#96751](https://github.com/grafana/grafana/pull/96751), [@matyax](https://github.com/matyax) +- **Loki:** Added support to show label types in Log Details [#97284](https://github.com/grafana/grafana/pull/97284), [@matyax](https://github.com/matyax) +- **Loki:** Allow regex in `label` derived field [#96609](https://github.com/grafana/grafana/pull/96609), [@svennergr](https://github.com/svennergr) +- **Loki:** Hide internal labels [#97323](https://github.com/grafana/grafana/pull/97323), [@svennergr](https://github.com/svennergr) +- **Loki:** Sync query direction with sort order in Explore and Dashboards [#98722](https://github.com/grafana/grafana/pull/98722), [@matyax](https://github.com/matyax) +- **OAuth:** Support client_secret_jwt for oauth providers when doing token exchange [#95455](https://github.com/grafana/grafana/pull/95455), [@naizerjohn-ms](https://github.com/naizerjohn-ms) +- **OAuth:** Use the attached external session data in OAuthToken and OAuthTokenSync [#96655](https://github.com/grafana/grafana/pull/96655), [@mgyongyosi](https://github.com/mgyongyosi) +- **Org Selection:** Show correct selected org when select is open [#96601](https://github.com/grafana/grafana/pull/96601), [@yincongcyincong](https://github.com/yincongcyincong) +- **PDF:** Add new zoom options (Enterprise) +- **Plugin Extensions:** Only load app plugins when necessary [#86624](https://github.com/grafana/grafana/pull/86624), [@leventebalogh](https://github.com/leventebalogh) +- **Plugins:** Add token to gcom requests [#96261](https://github.com/grafana/grafana/pull/96261), [@oshirohugo](https://github.com/oshirohugo) +- **Plugins:** Add token to gcom requests (Enterprise) +- **Plugins:** Disable version install when angular version is not supported [#97189](https://github.com/grafana/grafana/pull/97189), [@oshirohugo](https://github.com/oshirohugo) +- **Plugins:** Disable version installation for specific plugin types [#98597](https://github.com/grafana/grafana/pull/98597), [@oshirohugo](https://github.com/oshirohugo) +- **Plugins:** Update to latest go plugin SDK (v0.260.3) w/ arrow v18 [#97561](https://github.com/grafana/grafana/pull/97561), [@ryantxu](https://github.com/ryantxu) +- **Plugins:** Use grafana-com sso_api_token [#97096](https://github.com/grafana/grafana/pull/97096), [@oshirohugo](https://github.com/oshirohugo) +- **Plugins:** Use grafana-com sso_api_token (Enterprise) +- **Prometheus datasource:** Show info annotations in the UI [#97978](https://github.com/grafana/grafana/pull/97978), [@zenador](https://github.com/zenador) +- **Prometheus:** Improve handling of special chars in label values [#96067](https://github.com/grafana/grafana/pull/96067), [@NWRichmond](https://github.com/NWRichmond) +- **PublicDashboards:** Remove publicDashboards FF [#96578](https://github.com/grafana/grafana/pull/96578), [@juanicabanas](https://github.com/juanicabanas) +- **Reporting:** Add allow list email domain configuration (Enterprise) +- **Reporting:** Include the apiserver by default and deprecated internal ids (Enterprise) +- **RuntimeDataSource:** Support in core for runtime registered data sources [#93956](https://github.com/grafana/grafana/pull/93956), [@torkelo](https://github.com/torkelo) +- **SAML:** Add the ability to specify EntityID (Enterprise) +- **SAML:** Implement correct SLO with NameID and SessionIndex handling (Enterprise) +- **Security:** Update to Go 1.23.5 - Backport to v11.5.x [#99122](https://github.com/grafana/grafana/pull/99122), [@Proximyst](https://github.com/Proximyst) +- **Security:** Update to Go 1.23.5 - Backport to v11.5.x (Enterprise) +- **Snapshots:** Add RBAC roles for creating and deleting [#96126](https://github.com/grafana/grafana/pull/96126), [@evictorero](https://github.com/evictorero) +- **Storage:** Removes integration tests for MySQL 5.7 since it is EOL [#98013](https://github.com/grafana/grafana/pull/98013), [@inf0rmer](https://github.com/inf0rmer) +- **Tempo:** Add support for TraceQL Metrics exemplars [#96859](https://github.com/grafana/grafana/pull/96859), [@adrapereira](https://github.com/adrapereira) +- **Tempo:** Honor datasource TLS settings for gRPC requests [#97484](https://github.com/grafana/grafana/pull/97484), [@mdisibio](https://github.com/mdisibio) +- **Tempo:** Improve handling of multiple values in the Search tab query generation [#98427](https://github.com/grafana/grafana/pull/98427), [@adrapereira](https://github.com/adrapereira) +- **ToolbarButton:** Auto width on smaller screen sizes [#96023](https://github.com/grafana/grafana/pull/96023), [@yincongcyincong](https://github.com/yincongcyincong) +- **Trace View:** Set span filters as panel options [#98328](https://github.com/grafana/grafana/pull/98328), [@adrapereira](https://github.com/adrapereira) +- **TransformationFilter:** Implement RefID multi picker [#96841](https://github.com/grafana/grafana/pull/96841), [@Sergej-Vlasov](https://github.com/Sergej-Vlasov) +- **Transformations:** Add Delimiter format option to Extract fields [#97340](https://github.com/grafana/grafana/pull/97340), [@tskarhed](https://github.com/tskarhed) +- **Transformations:** Add RegExp option to Extract fields transformer [#96593](https://github.com/grafana/grafana/pull/96593), [@leeoniya](https://github.com/leeoniya) +- **Transformations:** GroupToMatrix add 0 as special value [#97642](https://github.com/grafana/grafana/pull/97642), [@tskarhed](https://github.com/tskarhed) +- **Zipkin:** Run queries through backend [#97754](https://github.com/grafana/grafana/pull/97754), [@ivanahuckova](https://github.com/ivanahuckova) + +### Bug fixes + +- **Alerting:** AlertingQueryRunner should skip descendant nodes of invalid queries [#97528](https://github.com/grafana/grafana/pull/97528), [@gillesdemey](https://github.com/gillesdemey) +- **Alerting:** Allow notification policy filters to match quoted matchers [#98525](https://github.com/grafana/grafana/pull/98525), [@gillesdemey](https://github.com/gillesdemey) +- **Alerting:** Fix alert rule list view summaries [#98433](https://github.com/grafana/grafana/pull/98433), [@yincongcyincong](https://github.com/yincongcyincong) +- **Alerting:** Fix alert rules unpausing after moving rule to different folder [#97580](https://github.com/grafana/grafana/pull/97580), [@santihernandezc](https://github.com/santihernandezc) +- **Alerting:** Fix ash not showing history graph in firefox [#98128](https://github.com/grafana/grafana/pull/98128), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Fix bug when saving a rule more than once [#96658](https://github.com/grafana/grafana/pull/96658), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Fix data-testid in RuleEditorSection [#97473](https://github.com/grafana/grafana/pull/97473), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Fix filtering rule group permissions based on their full path (Enterprise) +- **Alerting:** Fix go template parsing [#97145](https://github.com/grafana/grafana/pull/97145), [@konrad147](https://github.com/konrad147) +- **Alerting:** Fix label escaping in rule export [#97985](https://github.com/grafana/grafana/pull/97985), [@moustafab](https://github.com/moustafab) +- **Alerting:** Fix missing instances and history when Grafana rule is stored in folder with / [#97956](https://github.com/grafana/grafana/pull/97956), [@gillesdemey](https://github.com/gillesdemey) +- **Alerting:** Fix navigating to URLs with "%25" [#96992](https://github.com/grafana/grafana/pull/96992), [@gillesdemey](https://github.com/gillesdemey) +- **Alerting:** Fix no-change scenario in provisioning rule update API [#98389](https://github.com/grafana/grafana/pull/98389), [@alexander-akhmetov](https://github.com/alexander-akhmetov) +- **Alerting:** Fix not being able to remove a reducer when using range query [#97757](https://github.com/grafana/grafana/pull/97757), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Fix recording rules rendering simplified condition [#97497](https://github.com/grafana/grafana/pull/97497), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Fix removing reducer when inital value is instant [#97054](https://github.com/grafana/grafana/pull/97054), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Fix simplified query step [#97046](https://github.com/grafana/grafana/pull/97046), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Fix slack image uploading to use new api [#97817](https://github.com/grafana/grafana/pull/97817), [@moustafab](https://github.com/moustafab) +- **Alerting:** Fix terraform export of notification policy [#98429](https://github.com/grafana/grafana/pull/98429), [@moustafab](https://github.com/moustafab) +- **Alerting:** Fix updating condition when refId changes [#97753](https://github.com/grafana/grafana/pull/97753), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Alerting:** Fix using stacks- prefix instead of stack- for checking the namespace in boot data [#97492](https://github.com/grafana/grafana/pull/97492), [@soniaAguilarPeiron](https://github.com/soniaAguilarPeiron) +- **Anonymous User:** Adds validator service for anonymous users (Enterprise) +- **Auth:** Fix SAML user IsExternallySynced not being set correctly [#98487](https://github.com/grafana/grafana/pull/98487), [@volcanonoodle](https://github.com/volcanonoodle) +- **Azure Monitor:** Add safety around usage of frame.Meta.Custom struct [#97766](https://github.com/grafana/grafana/pull/97766), [@adamyeats](https://github.com/adamyeats) +- **Azure/GCM:** Improve error display [#96921](https://github.com/grafana/grafana/pull/96921), [@aangelisc](https://github.com/aangelisc) +- **CloudWatch:** Fix conditions for fetching wildcards [#98648](https://github.com/grafana/grafana/pull/98648), [@iwysiu](https://github.com/iwysiu) +- **CloudWatch:** Fix interpolation of log groups when fetching fields [#98054](https://github.com/grafana/grafana/pull/98054), [@idastambuk](https://github.com/idastambuk) +- **Dashboard:** Fixes issue with compatability of old DashboardModel.annotations [#97328](https://github.com/grafana/grafana/pull/97328), [@torkelo](https://github.com/torkelo) +- **Dashboards:** Fix issue where filtered panels would not react to variable changes [#98718](https://github.com/grafana/grafana/pull/98718), [@oscarkilhed](https://github.com/oscarkilhed) +- **Dashboards:** Fixes week relative time ranges when weekStart was changed [#98167](https://github.com/grafana/grafana/pull/98167), [@torkelo](https://github.com/torkelo) +- **Dashboards:** Panel react for `timeFrom` and `timeShift` changes using variables [#98510](https://github.com/grafana/grafana/pull/98510), [@Sergej-Vlasov](https://github.com/Sergej-Vlasov) +- **DateTimePicker:** Fixes issue with date picker showing invalid date [#97888](https://github.com/grafana/grafana/pull/97888), [@torkelo](https://github.com/torkelo) +- **Fix:** Add support for datasource variable queries [#98098](https://github.com/grafana/grafana/pull/98098), [@sunker](https://github.com/sunker) +- **Fix:** Do not fetch Orgs if the user is authenticated by apikey/sa or render key [#97162](https://github.com/grafana/grafana/pull/97162), [@mgyongyosi](https://github.com/mgyongyosi) +- **Fix:** Double encoding of URLs when using data proxy [#98494](https://github.com/grafana/grafana/pull/98494), [@s4kh](https://github.com/s4kh) +- **Font:** Disable contextual font ligatures [#98521](https://github.com/grafana/grafana/pull/98521), [@ashharrison90](https://github.com/ashharrison90) +- **GrafanaUI:** Fix inconsistent controlled/uncontrolled state in AutoSizeInput [#96696](https://github.com/grafana/grafana/pull/96696), [@joshhunt](https://github.com/joshhunt) +- **GrafanaUI:** Revert: Fix inconsistent controlled/uncontrolled state in AutoSizeInput [#97551](https://github.com/grafana/grafana/pull/97551), [@itsmylife](https://github.com/itsmylife) +- **InfluxDB:** Adhoc filters can use template vars as values [#98567](https://github.com/grafana/grafana/pull/98567), [@bossinc](https://github.com/bossinc) +- **Library Panel:** Fix issue where library panels did not display panel links. [#98655](https://github.com/grafana/grafana/pull/98655), [@yincongcyincong](https://github.com/yincongcyincong) +- **LibraryPanel:** Fallback to panel title if library panel title is not set [#99411](https://github.com/grafana/grafana/pull/99411), [@ivanortegaalba](https://github.com/ivanortegaalba) +- **Loki:** Fix a bug when reading frames without values but warnings [#97197](https://github.com/grafana/grafana/pull/97197), [@svennergr](https://github.com/svennergr) +- **Loki:** Only hide a set of labels instead of every label starting with `__` [#98730](https://github.com/grafana/grafana/pull/98730), [@svennergr](https://github.com/svennergr) +- **Org:** Fix redirection logic to work consistently [#96521](https://github.com/grafana/grafana/pull/96521), [@yincongcyincong](https://github.com/yincongcyincong) +- **Panel inspect:** Fix file names of data download included uninterpolated variable names. [#98832](https://github.com/grafana/grafana/pull/98832), [@alexrosenfeld10](https://github.com/alexrosenfeld10) +- **Scenes:** Upgrade to 5.36.3 [#98661](https://github.com/grafana/grafana/pull/98661), [@ivanortegaalba](https://github.com/ivanortegaalba) +- **Snapshot:** Show proper breadcrumb path [#98806](https://github.com/grafana/grafana/pull/98806), [@ashharrison90](https://github.com/ashharrison90) +- **Time Picker:** Fix "Fiscal year start month" selection behaviour [#98576](https://github.com/grafana/grafana/pull/98576), [@ashharrison90](https://github.com/ashharrison90) +- **Unified Storage:** Add support for verify-full in postgres [#96825](https://github.com/grafana/grafana/pull/96825), [@chaudyg](https://github.com/chaudyg) +- **Unified Storage:** Use tls preferred when grafana db using ssl [#97378](https://github.com/grafana/grafana/pull/97378), [@owensmallwood](https://github.com/owensmallwood) +- **Usage Insights:** Fix usage insight errors being logged as [object Object] [#93502](https://github.com/grafana/grafana/pull/93502), [@mmandrus](https://github.com/mmandrus) + +### Breaking changes + +- **Loki:** Default to `/labels` API with `query` param instead of `/series` API [#97935](https://github.com/grafana/grafana/pull/97935), [@svennergr](https://github.com/svennergr) + +### Plugin development fixes & changes + +- **Grafana UI:** Re-add react-router-dom as a dependency [#97540](https://github.com/grafana/grafana/pull/97540), [@leventebalogh](https://github.com/leventebalogh) + + # 11.4.1 (2025-01-28) diff --git a/docs/sources/datasources/pyroscope/configure-pyroscope-data-source.md b/docs/sources/datasources/pyroscope/configure-pyroscope-data-source.md index 8a344f6a7e6..6f6b30ad045 100644 --- a/docs/sources/datasources/pyroscope/configure-pyroscope-data-source.md +++ b/docs/sources/datasources/pyroscope/configure-pyroscope-data-source.md @@ -33,11 +33,11 @@ refs: destination: /docs/grafana//datasources/tempo/configure-tempo-data-source/ - pattern: /docs/grafana-cloud/ destination: docs/grafana-cloud/connect-externally-hosted/data-sources/tempo/configure-tempo-data-source/ - provisioning-data-sources: + explore-profiles: - pattern: /docs/grafana/ - destination: /docs/grafana//administration/provisioning/#data-sources - - pattern: /docs/grafana-cloud/provision - destination: /docs/grafana//administration/provisioning/#data-sources + destination: /docs/grafana//explore/simplified-exploration/profiles/ + - pattern: /docs/grafana-cloud/ + destination: /docs/grafana-cloud/visualizations/simplified-exploration/profiles/ --- # Configure the Grafana Pyroscope data source @@ -73,7 +73,7 @@ To configure basic settings for the data source, complete the following steps: 1. On the **Settings** tab, complete the **Name**, **Connection**, and **Authentication** sections. - Use the **Name** field to specify the name used for the data source in panels, queries, and Explore. Toggle the **Default** switch for the data source to be pre-selected for new panels. -- Under **Connection**, enter the **URL** of the Pyroscope instance. For example, `https://example.com:4100`. +- Under **Connection**, enter the **URL** of the Pyroscope instance. For example, `https://example.com:4100`. Refer to [Connection URL](#connection-url) for more information. - Complete the [**Authentication** section](#authentication). 1. Optional: Use **Additional settings** to configure other options. @@ -89,6 +89,24 @@ To modify an existing Pyroscope data source: 1. Optional: Use **Additional settings** to configure or modify other options. 1. After completing your updates, select **Save & test**. +#### Connection URL + +The data source connection URL should point to a location of a running Pyroscope backend. + +**Grafana Cloud Profiles** + +Your Grafana Cloud instance automatically includes a fully provisioned data source. + +If you are running a self-managed Grafana instance or need to configure an additional Pyroscope data source pointing to Grafana Cloud Profiles, you can find the Pyroscope URL under the **Manage your stack** section for your organization. + +**Self-managed Pyroscope backend** + +The connection URL for a self-managed Pyroscope backend depends on how Pyroscope is deployed. +Refer to the steps under [Query profiles in Grafana](https://grafana.com/docs/pyroscope//deploy-kubernetes/helm/#query-profiles-in-grafana) for more information on how to configure the data source. + +If you plan to use the [Explore Profiles](ref:explore-profiles) application and you are running a self-managed Pyroscope backend in microservices mode, the data source connection URL should point to a gateway or proxy that routes requests to the corresponding Pyroscope service. +Refer to the [Helm ingress configuration](https://github.com/grafana/pyroscope/blob/main/operations/pyroscope/helm/pyroscope/templates/ingress.yaml) for specific routing requirements. + ## Authentication Use this section to select an authentication method to access the data source. diff --git a/docs/sources/datasources/pyroscope/query-profile-data.md b/docs/sources/datasources/pyroscope/query-profile-data.md index d56a629d42d..5f391efe0b7 100644 --- a/docs/sources/datasources/pyroscope/query-profile-data.md +++ b/docs/sources/datasources/pyroscope/query-profile-data.md @@ -33,7 +33,7 @@ refs: destination: /docs/grafana//explore/simplified-exploration/profiles/ - pattern: /docs/grafana-cloud/ destination: /docs/grafana-cloud/visualizations/simplified-exploration/profiles/ - explore-profile-install: + explore-profiles-install: - pattern: /docs/grafana/ destination: /docs/grafana//explore/simplified-exploration/profiles/access/ - pattern: /docs/grafana-cloud/ diff --git a/docs/sources/panels-visualizations/visualizations/canvas/index.md b/docs/sources/panels-visualizations/visualizations/canvas/index.md index ebd7162047e..b31b8018902 100644 --- a/docs/sources/panels-visualizations/visualizations/canvas/index.md +++ b/docs/sources/panels-visualizations/visualizations/canvas/index.md @@ -258,6 +258,13 @@ You can enable panning and zooming in a canvas. This allows you to both create a {{< docs/public-preview product="Canvas pan and zoom" featureFlag="`canvasPanelPanZoom`" >}} +Use the following pointer and keyboard strokes: + +- **Zoom in** - Scroll up +- **Zoom out** - Scroll down +- **Pan** - Middle mouse/wheel + drag OR Control + right-click + drag +- **Reset** - Double-click + {{< video-embed src="/media/docs/grafana/2024-01-05-Canvas-Pan-&-Zoom-Enablement-Video.mp4" max-width="750px" alt="Canvas pan and zoom enablement video" >}} ##### Infinite panning diff --git a/pkg/api/folder.go b/pkg/api/folder.go index eb4cb853dda..185eb427d7d 100644 --- a/pkg/api/folder.go +++ b/pkg/api/folder.go @@ -412,6 +412,7 @@ func (hs *HTTPServer) newToFolderDto(c *contextmodel.ReqContext, f *folder.Folde return dtos.Folder{ ID: f.ID, // nolint:staticcheck UID: f.UID, + OrgID: f.OrgID, Title: f.Title, URL: f.URL, HasACL: f.HasACL, diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index 4847f66e2f4..92cff840635 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -801,13 +801,9 @@ func (hs *HTTPServer) getDefaultCiphers(tlsVersion uint16, protocol string) []ui tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, tls.TLS_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, } } if protocol == "h2" { diff --git a/pkg/apimachinery/identity/context.go b/pkg/apimachinery/identity/context.go index ae71f06eaf1..b2266899861 100644 --- a/pkg/apimachinery/identity/context.go +++ b/pkg/apimachinery/identity/context.go @@ -71,6 +71,8 @@ var serviceIdentityPermissions = getWildcardPermissions( "dashboards:write", "dashboards:create", "datasources:read", + "alert.provisioning:write", + "alert.provisioning.secrets:read", ) func IsServiceIdentity(ctx context.Context) bool { diff --git a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go index f3b86f8b639..6c1553f9bad 100644 --- a/pkg/registry/apis/dashboard/legacy/sql_dashboards.go +++ b/pkg/registry/apis/dashboard/legacy/sql_dashboards.go @@ -102,9 +102,9 @@ func (a *dashboardSqlAccess) getRows(ctx context.Context, sql *legacysql.LegacyD return nil, fmt.Errorf("execute template %q: %w", tmpl.Name(), err) } q := rawQuery - // if true { - // pretty := sqltemplate.RemoveEmptyLines(rawQuery) - // fmt.Printf("DASHBOARD QUERY: %s [%+v] // %+v\n", pretty, req.GetArgs(), query) + // if false { + // pretty := sqltemplate.RemoveEmptyLines(rawQuery) + // fmt.Printf("DASHBOARD QUERY: %s [%+v] // %+v\n", pretty, req.GetArgs(), query) // } rows, err := sql.DB.GetSqlxSession().Query(ctx, q, req.GetArgs()...) @@ -132,12 +132,16 @@ type rowsWrapper struct { a *dashboardSqlAccess rows *sql.Rows history bool + count int canReadDashboard func(scopes ...string) bool // Current row *dashboardRow err error + + // max 100 rejected? + rejected []dashboardRow } func (a *dashboardSqlAccess) GetResourceStats(ctx context.Context, namespace string, minCount int) ([]resource.ResourceStats, error) { @@ -159,10 +163,16 @@ func (r *rowsWrapper) Next() bool { // breaks after first readable value for r.rows.Next() { + r.count++ + r.row, err = r.a.scanRow(r.rows, r.history) if err != nil { - r.err = err - return false + if len(r.rejected) > 1000 || r.row == nil { + r.err = fmt.Errorf("too many rejected rows (%d) %w", len(r.rejected), err) + return false + } + r.rejected = append(r.rejected, *r.row) + continue } if r.row != nil { @@ -177,7 +187,7 @@ func (r *rowsWrapper) Next() bool { continue } - // returns the first folder it can + // returns the first visible dashboard return true } } @@ -245,7 +255,7 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRo var createdByID sql.NullInt64 var message sql.NullString - var plugin_id string + var plugin_id sql.NullString var origin_name sql.NullString var origin_path sql.NullString var origin_ts sql.NullInt64 @@ -319,17 +329,17 @@ func (a *dashboardSqlAccess) scanRow(rows *sql.Rows, history bool) (*dashboardRo repo.Path = originPath } meta.SetRepositoryInfo(repo) - } else if plugin_id != "" { + } else if plugin_id.String != "" { meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{ Name: "plugin", - Path: plugin_id, + Path: plugin_id.String, }) } if len(data) > 0 { err = dash.Spec.UnmarshalJSON(data) if err != nil { - return row, err + return row, fmt.Errorf("JSON unmarshal error for: %s // %w", dash.Name, err) } } dash.Spec.Remove("id") @@ -453,12 +463,12 @@ func (a *dashboardSqlAccess) GetLibraryPanels(ctx context.Context, query Library return nil, fmt.Errorf("expected non zero orgID") } - sql, err := a.sql(ctx) + sqlx, err := a.sql(ctx) if err != nil { return nil, err } - req := newLibraryQueryReq(sql, &query) + req := newLibraryQueryReq(sqlx, &query) rawQuery, err := sqltemplate.Execute(sqlQueryPanels, req) if err != nil { return nil, fmt.Errorf("execute template %q: %w", sqlQueryPanels.Name(), err) @@ -466,7 +476,7 @@ func (a *dashboardSqlAccess) GetLibraryPanels(ctx context.Context, query Library q := rawQuery res := &dashboard.LibraryPanelList{} - rows, err := sql.DB.GetSqlxSession().Query(ctx, q, req.GetArgs()...) + rows, err := sqlx.DB.GetSqlxSession().Query(ctx, q, req.GetArgs()...) defer func() { if rows != nil { _ = rows.Close() @@ -479,7 +489,7 @@ func (a *dashboardSqlAccess) GetLibraryPanels(ctx context.Context, query Library type panel struct { ID int64 UID string - FolderUID string + FolderUID sql.NullString Created time.Time CreatedBy string @@ -551,7 +561,9 @@ func (a *dashboardSqlAccess) GetLibraryPanels(ctx context.Context, query Library if err != nil { return nil, err } - meta.SetFolder(p.FolderUID) + if p.FolderUID.Valid { + meta.SetFolder(p.FolderUID.String) + } meta.SetCreatedBy(p.CreatedBy) meta.SetGeneration(1) meta.SetDeprecatedInternalID(p.ID) //nolint:staticcheck @@ -570,7 +582,7 @@ func (a *dashboardSqlAccess) GetLibraryPanels(ctx context.Context, query Library } } if query.UID == "" { - rv, err := sql.GetResourceVersion(ctx, "library_element", "updated") + rv, err := sqlx.GetResourceVersion(ctx, "library_element", "updated") if err == nil { res.ResourceVersion = strconv.FormatInt(rv, 10) } diff --git a/pkg/registry/apis/dashboard/legacysearcher/search_client.go b/pkg/registry/apis/dashboard/legacysearcher/search_client.go index ac221ba86e2..17aa5aa16a9 100644 --- a/pkg/registry/apis/dashboard/legacysearcher/search_client.go +++ b/pkg/registry/apis/dashboard/legacysearcher/search_client.go @@ -8,6 +8,7 @@ import ( claims "github.com/grafana/authlib/types" "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/sqlstore/searchstore" "github.com/grafana/grafana/pkg/storage/unified/resource" "google.golang.org/grpc" ) @@ -48,6 +49,7 @@ func (c *DashboardSearchClient) Search(ctx context.Context, req *resource.Resour Title: req.Query, Limit: req.Limit, // FolderUIDs: req.FolderUIDs, + Type: searchstore.TypeDashboard, SignedInUser: user, } diff --git a/pkg/registry/apis/dashboard/search_test.go b/pkg/registry/apis/dashboard/search_test.go index 23d7ad18278..fbd3a209ff5 100644 --- a/pkg/registry/apis/dashboard/search_test.go +++ b/pkg/registry/apis/dashboard/search_test.go @@ -7,15 +7,17 @@ import ( "testing" "github.com/grafana/grafana/pkg/apimachinery/identity" - "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/services/user" - "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/resource" "google.golang.org/grpc" ) +/* +Search Fallback was returning both Folders and Dashboards which resulted +in issues with rendering the Folder UI. Also, filters are not implemented +yet. For those reasons, we will be disabling Search Fallback for now func TestSearchFallback(t *testing.T) { t.Run("should hit legacy search handler on mode 0", func(t *testing.T) { mockClient := &MockClient{} @@ -173,6 +175,7 @@ func TestSearchFallback(t *testing.T) { } }) } +*/ func TestSearchHandlerFields(t *testing.T) { // Create a mock client diff --git a/pkg/server/server.go b/pkg/server/server.go index ca1263d9e4d..a56b6c9f184 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -131,7 +131,7 @@ func (s *Server) Init() error { return err } - return s.provisioningService.RunInitProvisioners(s.context) + return nil } // Run initializes and starts services. This will block until all services have diff --git a/pkg/services/dashboards/service/dashboard_service.go b/pkg/services/dashboards/service/dashboard_service.go index ad78d14952f..8f4953d2723 100644 --- a/pkg/services/dashboards/service/dashboard_service.go +++ b/pkg/services/dashboards/service/dashboard_service.go @@ -1689,9 +1689,9 @@ func (dr *DashboardServiceImpl) searchDashboardsThroughK8sRaw(ctx context.Contex // but is currently not needed by other services in the backend if query.Title != "" { req := []*resource.Requirement{{ - Key: resource.SEARCH_FIELD_TITLE, + Key: resource.SEARCH_FIELD_TITLE_SORT, // use title sort to prevent issues with `-` in the title & how bleve searches Operator: string(selection.In), - Values: []string{query.Title}, + Values: []string{strings.ToLower(query.Title)}, }} request.Options.Fields = append(request.Options.Fields, req...) } diff --git a/pkg/services/dashboardversion/dashverimpl/dashver.go b/pkg/services/dashboardversion/dashverimpl/dashver.go index 02bed95b96e..98406b6a7b4 100644 --- a/pkg/services/dashboardversion/dashverimpl/dashver.go +++ b/pkg/services/dashboardversion/dashverimpl/dashver.go @@ -283,6 +283,15 @@ func (s *Service) UnstructuredToLegacyDashboardVersion(ctx context.Context, item return nil, err } + // if updated by is set, then this version of the dashboard was "created" + // by that user + if obj.GetUpdatedBy() != "" { + updatedBy, err := s.k8sclient.GetUserFromMeta(ctx, obj.GetUpdatedBy()) + if err == nil && updatedBy != nil { + createdBy = updatedBy + } + } + id, err := obj.GetResourceVersionInt64() if err != nil { return nil, err diff --git a/pkg/services/dashboardversion/dashverimpl/dashver_test.go b/pkg/services/dashboardversion/dashverimpl/dashver_test.go index ed58c27bb7f..5d71b3457d1 100644 --- a/pkg/services/dashboardversion/dashverimpl/dashver_test.go +++ b/pkg/services/dashboardversion/dashverimpl/dashver_test.go @@ -39,7 +39,7 @@ func TestDashboardVersionService(t *testing.T) { require.Equal(t, dashboard.ToDTO("uid"), dashboardVersion) }) - t.Run("Get dashboard version through k8s", func(t *testing.T) { + t.Run("Get dashboard versions through k8s", func(t *testing.T) { dashboardService := dashboards.NewFakeDashboardService(t) dashboardVersionService := Service{dashSvc: dashboardService, features: featuremgmt.WithFeatures()} mockCli := new(client.MockK8sHandler) @@ -47,7 +47,7 @@ func TestDashboardVersionService(t *testing.T) { dashboardVersionService.features = featuremgmt.WithFeatures(featuremgmt.FlagKubernetesCliDashboards) dashboardService.On("GetDashboardUIDByID", mock.Anything, mock.AnythingOfType("*dashboards.GetDashboardRefByIDQuery")).Return(&dashboards.DashboardRef{UID: "uid"}, nil) - mockCli.On("GetUserFromMeta", mock.Anything, mock.Anything).Return(&user.User{}, nil) + mockCli.On("GetUserFromMeta", mock.Anything, "user:1").Return(&user.User{ID: 1}, nil) mockCli.On("Get", mock.Anything, "uid", int64(1), v1.GetOptions{ResourceVersion: "10"}, mock.Anything).Return(&unstructured.Unstructured{ Object: map[string]any{ "metadata": map[string]any{ @@ -56,6 +56,9 @@ func TestDashboardVersionService(t *testing.T) { "labels": map[string]any{ utils.LabelKeyDeprecatedInternalID: "42", // nolint:staticcheck }, + "annotations": map[string]any{ + utils.AnnoKeyCreatedBy: "user:1", + }, }, "spec": map[string]any{ "version": int64(10), @@ -73,8 +76,43 @@ func TestDashboardVersionService(t *testing.T) { ParentVersion: 9, DashboardID: 42, DashboardUID: "uid", + CreatedBy: 1, Data: simplejson.NewFromAny(map[string]any{"uid": "uid", "version": int64(10)}), }) + + mockCli.On("GetUserFromMeta", mock.Anything, "user:2").Return(&user.User{ID: 2}, nil) + mockCli.On("Get", mock.Anything, "uid", int64(1), v1.GetOptions{ResourceVersion: "11"}, mock.Anything).Return(&unstructured.Unstructured{ + Object: map[string]any{ + "metadata": map[string]any{ + "name": "uid", + "resourceVersion": "11", + "labels": map[string]any{ + utils.LabelKeyDeprecatedInternalID: "42", // nolint:staticcheck + }, + "annotations": map[string]any{ + utils.AnnoKeyCreatedBy: "user:1", + utils.AnnoKeyUpdatedBy: "user:2", // if updated by is set, that is the version creator + }, + }, + "spec": map[string]any{ + "version": int64(11), + }, + }}, nil).Once() + res, err = dashboardVersionService.Get(context.Background(), &dashver.GetDashboardVersionQuery{ + DashboardID: 42, + OrgID: 1, + Version: 11, + }) + require.Nil(t, err) + require.Equal(t, res, &dashver.DashboardVersionDTO{ + ID: 11, // RV should be used + Version: 11, + ParentVersion: 10, + DashboardID: 42, + DashboardUID: "uid", + CreatedBy: 2, + Data: simplejson.NewFromAny(map[string]any{"uid": "uid", "version": int64(11)}), + }) }) } diff --git a/pkg/services/folder/folderimpl/folder_unifiedstorage.go b/pkg/services/folder/folderimpl/folder_unifiedstorage.go index 3fe81899597..ae1f579efd0 100644 --- a/pkg/services/folder/folderimpl/folder_unifiedstorage.go +++ b/pkg/services/folder/folderimpl/folder_unifiedstorage.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/apimachinery/utils" + dashboardv0 "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" "github.com/grafana/grafana/pkg/apis/folder/v0alpha1" "github.com/grafana/grafana/pkg/events" "github.com/grafana/grafana/pkg/infra/metrics" @@ -106,27 +107,24 @@ func (s *Service) getFromApiServer(ctx context.Context, q *folder.GetFolderQuery var dashFolder *folder.Folder var err error switch { - case q.UID != nil: - if *q.UID == "" { - return &folder.GeneralFolder, nil - } + case q.UID != nil && *q.UID != "": dashFolder, err = s.unifiedStore.Get(ctx, *q) if err != nil { return nil, toFolderError(err) } // nolint:staticcheck - case q.ID != nil: + case q.ID != nil && *q.ID != 0: dashFolder, err = s.getFolderByIDFromApiServer(ctx, *q.ID, q.OrgID) if err != nil { return nil, toFolderError(err) } - case q.Title != nil: + case q.Title != nil && *q.Title != "": dashFolder, err = s.getFolderByTitleFromApiServer(ctx, q.OrgID, *q.Title, q.ParentUID) if err != nil { return nil, toFolderError(err) } default: - return nil, folder.ErrBadRequest.Errorf("either on of UID, ID, Title fields must be present") + return &folder.GeneralFolder, nil } if dashFolder.IsGeneral() { @@ -242,9 +240,9 @@ func (s *Service) getFolderByTitleFromApiServer(ctx context.Context, orgID int64 Key: folderkey, Fields: []*resource.Requirement{ { - Key: resource.SEARCH_FIELD_TITLE, + Key: resource.SEARCH_FIELD_TITLE_SORT, Operator: string(selection.In), - Values: []string{title}, + Values: []string{strings.ToLower(title)}, }, }, Labels: []*resource.Requirement{}, @@ -590,16 +588,54 @@ func (s *Service) deleteFromApiServer(ctx context.Context, cmd *folder.DeleteFol // if dashboard restore is on we don't delete public dashboards, the hard delete will take care of it later if !s.features.IsEnabledGlobally(featuremgmt.FlagDashboardRestore) { // We need a list of dashboard uids inside the folder to delete related public dashboards - dashes, err := s.dashboardStore.FindDashboards(ctx, &dashboards.FindPersistedDashboardsQuery{SignedInUser: cmd.SignedInUser, FolderUIDs: folders, OrgId: cmd.OrgID}) - if err != nil { - return folder.ErrInternal.Errorf("failed to fetch dashboards: %w", err) - } + var dashboardUIDs []string + // we cannot use the dashboard service directly due to circular dependencies, + // so either use the search client if the feature is enabled or use the dashboard store + if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesCliDashboards) { + dashboardKey := &resource.ResourceKey{ + Namespace: s.k8sclient.getNamespace(cmd.OrgID), + Group: dashboardv0.DashboardResourceInfo.GroupVersionResource().Group, + Resource: dashboardv0.DashboardResourceInfo.GroupVersionResource().Resource, + } + request := &resource.ResourceSearchRequest{ + Options: &resource.ListOptions{ + Key: dashboardKey, + Labels: []*resource.Requirement{}, + Fields: []*resource.Requirement{ + { + Key: resource.SEARCH_FIELD_FOLDER, + Operator: string(selection.In), + Values: folders, + }, + }, + }, + Limit: 100000} - dashboardUIDs := make([]string, 0, len(dashes)) - for _, dashboard := range dashes { - dashboardUIDs = append(dashboardUIDs, dashboard.UID) - } + client := s.k8sclient.getSearcher(ctx) + res, err := client.Search(ctx, request) + if err != nil { + return folder.ErrInternal.Errorf("failed to fetch dashboards: %w", err) + } + hits, err := dashboardsearch.ParseResults(res, 0) + if err != nil { + return folder.ErrInternal.Errorf("failed to fetch dashboards: %w", err) + } + + dashboardUIDs = make([]string, len(hits.Hits)) + for i, dashboard := range hits.Hits { + dashboardUIDs[i] = dashboard.Name + } + } else { + dashes, err := s.dashboardStore.FindDashboards(ctx, &dashboards.FindPersistedDashboardsQuery{SignedInUser: cmd.SignedInUser, FolderUIDs: folders, OrgId: cmd.OrgID}) + if err != nil { + return folder.ErrInternal.Errorf("failed to fetch dashboards: %w", err) + } + dashboardUIDs = make([]string, len(dashes)) + for i, dashboard := range dashes { + dashboardUIDs[i] = dashboard.UID + } + } // Delete all public dashboards in the folders err = s.publicDashboardService.DeleteByDashboardUIDs(ctx, cmd.OrgID, dashboardUIDs) if err != nil { diff --git a/pkg/services/folder/folderimpl/folder_unifiedstorage_test.go b/pkg/services/folder/folderimpl/folder_unifiedstorage_test.go index 3d5a84a6aad..a19c83f209e 100644 --- a/pkg/services/folder/folderimpl/folder_unifiedstorage_test.go +++ b/pkg/services/folder/folderimpl/folder_unifiedstorage_test.go @@ -424,9 +424,11 @@ func TestIntegrationFolderServiceViaUnifiedStorage(t *testing.T) { require.NoError(t, err) }) - t.Run("When get folder by ID should return folder", func(t *testing.T) { + t.Run("When get folder by ID and uid is an empty string should return folder by id", func(t *testing.T) { id := int64(123) + emptyString := "" query := &folder.GetFolderQuery{ + UID: &emptyString, ID: &id, OrgID: 1, SignedInUser: usr, @@ -482,10 +484,13 @@ func TestIntegrationFolderServiceViaUnifiedStorage(t *testing.T) { }) t.Run("Returns root folder", func(t *testing.T) { - t.Run("When the folder UID is blank should return the root folder", func(t *testing.T) { + t.Run("When the folder UID and title are blank, and id is 0, should return the root folder", func(t *testing.T) { emptyString := "" + idZero := int64(0) actual, err := folderService.Get(ctx, &folder.GetFolderQuery{ UID: &emptyString, + ID: &idZero, + Title: &emptyString, OrgID: 1, SignedInUser: usr, }) @@ -562,7 +567,7 @@ func (r resourceClientMock) Search(ctx context.Context, in *resource.ResourceSea } if len(in.Options.Fields) > 0 && - in.Options.Fields[0].Key == resource.SEARCH_FIELD_TITLE && + in.Options.Fields[0].Key == resource.SEARCH_FIELD_TITLE_SORT && in.Options.Fields[0].Operator == "in" && len(in.Options.Fields[0].Values) > 0 && in.Options.Fields[0].Values[0] == "foo" { diff --git a/pkg/services/ngalert/schedule/alert_rule.go b/pkg/services/ngalert/schedule/alert_rule.go index f4a301aff78..85c254b0a59 100644 --- a/pkg/services/ngalert/schedule/alert_rule.go +++ b/pkg/services/ngalert/schedule/alert_rule.go @@ -352,16 +352,23 @@ func (a *alertRule) Run() error { case <-grafanaCtx.Done(): reason := grafanaCtx.Err() - // clean up the state only if the reason for stopping the evaluation loop is that the rule was deleted + + // We do not want a context to be unbounded which could potentially cause a go routine running + // indefinitely. 1 minute is an almost randomly chosen timeout, big enough to cover the majority of the + // cases. + ctx, cancelFunc := context.WithTimeout(context.Background(), time.Minute) + defer cancelFunc() + if errors.Is(reason, errRuleDeleted) { - // We do not want a context to be unbounded which could potentially cause a go routine running - // indefinitely. 1 minute is an almost randomly chosen timeout, big enough to cover the majority of the - // cases. - ctx, cancelFunc := context.WithTimeout(context.Background(), time.Minute) - defer cancelFunc() - states := a.stateManager.DeleteStateByRuleUID(ngmodels.WithRuleKey(ctx, a.key.AlertRuleKey), a.key, ngmodels.StateReasonRuleDeleted) - a.expireAndSend(grafanaCtx, states) + // Clean up the state and send resolved notifications for firing alerts only if the reason for stopping + // the evaluation loop is that the rule was deleted. + stateTransitions := a.stateManager.DeleteStateByRuleUID(ngmodels.WithRuleKey(ctx, a.key.AlertRuleKey), a.key, ngmodels.StateReasonRuleDeleted) + a.expireAndSend(grafanaCtx, stateTransitions) + } else { + // Otherwise, just clean up the cache. + a.stateManager.ForgetStateByRuleUID(ngmodels.WithRuleKey(ctx, a.key.AlertRuleKey), a.key) } + a.logger.Debug("Stopping alert rule routine", "reason", reason) return nil } diff --git a/pkg/services/ngalert/schedule/alert_rule_test.go b/pkg/services/ngalert/schedule/alert_rule_test.go index 4593a4052e0..02ea0dd5735 100644 --- a/pkg/services/ngalert/schedule/alert_rule_test.go +++ b/pkg/services/ngalert/schedule/alert_rule_test.go @@ -3,6 +3,7 @@ package schedule import ( "bytes" "context" + "errors" "fmt" "math" "math/rand" @@ -278,7 +279,12 @@ func TestAlertRuleIdentifier(t *testing.T) { } func blankRuleForTests(ctx context.Context, key models.AlertRuleKeyWithGroup) *alertRule { - return newAlertRule(ctx, key, nil, false, 0, nil, nil, nil, nil, nil, nil, log.NewNopLogger(), nil, nil, nil) + managerCfg := state.ManagerCfg{ + Historian: &state.FakeHistorian{}, + Log: log.NewNopLogger(), + } + st := state.NewManager(managerCfg, state.NewNoopPersister()) + return newAlertRule(ctx, key, nil, false, 0, nil, st, nil, nil, nil, nil, log.NewNopLogger(), nil, nil, nil) } func TestRuleRoutine(t *testing.T) { @@ -477,12 +483,26 @@ func TestRuleRoutine(t *testing.T) { } t.Run("should exit", func(t *testing.T) { - t.Run("and not clear the state if parent context is cancelled", func(t *testing.T) { - stoppedChan := make(chan error) - sch, _, _, _ := createSchedule(make(chan time.Time), nil) + rule := gen.With(withQueryForState(t, eval.Alerting)).GenerateRef() + genEvalResults := func(now time.Time) eval.Results { + return eval.GenerateResults( + rand.Intn(5)+1, + eval.ResultGen( + eval.WithEvaluatedAt(now), + // State should be alerting to test resolved notifications in some cases. + // When the alert rule is firing and is deleted, we should send + // resolved notifications. + eval.WithState(eval.Alerting), + ), + ) + } - rule := gen.GenerateRef() - _ = sch.stateManager.ProcessEvalResults(context.Background(), sch.clock.Now(), rule, eval.GenerateResults(rand.Intn(5)+1, eval.ResultGen(eval.WithEvaluatedAt(sch.clock.Now()))), nil, nil) + t.Run("and clean up the state if parent context is cancelled", func(t *testing.T) { + stoppedChan := make(chan error) + sender := NewSyncAlertsSenderMock() + sch, _, _, _ := createSchedule(make(chan time.Time), sender) + + _ = sch.stateManager.ProcessEvalResults(context.Background(), sch.clock.Now(), rule, genEvalResults(sch.clock.Now()), nil, nil) expectedStates := sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID) require.NotEmpty(t, expectedStates) @@ -497,14 +517,40 @@ func TestRuleRoutine(t *testing.T) { cancel() err := waitForErrChannel(t, stoppedChan) require.NoError(t, err) - require.Equal(t, len(expectedStates), len(sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID))) + require.Empty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID)) + sender.AlertsSenderMock.AssertNotCalled(t, "Send") }) - t.Run("and clean up the state if delete is cancellation reason for inner context", func(t *testing.T) { - stoppedChan := make(chan error) - sch, _, _, _ := createSchedule(make(chan time.Time), nil) - rule := gen.GenerateRef() - _ = sch.stateManager.ProcessEvalResults(context.Background(), sch.clock.Now(), rule, eval.GenerateResults(rand.Intn(5)+1, eval.ResultGen(eval.WithEvaluatedAt(sch.clock.Now()))), nil, nil) + t.Run("and clean up the state but not send anything if the reason is not rule deleted", func(t *testing.T) { + stoppedChan := make(chan error) + sender := NewSyncAlertsSenderMock() + sch, _, _, _ := createSchedule(make(chan time.Time), sender) + + _ = sch.stateManager.ProcessEvalResults(context.Background(), sch.clock.Now(), rule, genEvalResults(sch.clock.Now()), nil, nil) + require.NotEmpty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID)) + + factory := ruleFactoryFromScheduler(sch) + ruleInfo := factory.new(context.Background(), rule) + go func() { + err := ruleInfo.Run() + stoppedChan <- err + }() + + ruleInfo.Stop(errors.New("some reason")) + err := waitForErrChannel(t, stoppedChan) + require.NoError(t, err) + + require.Empty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID)) + sender.AlertsSenderMock.AssertNotCalled(t, "Send") + }) + + t.Run("and send resolved notifications if errRuleDeleted is the reason for stopping", func(t *testing.T) { + stoppedChan := make(chan error) + sender := NewSyncAlertsSenderMock() + sender.EXPECT().Send(mock.Anything, mock.Anything, mock.Anything).Times(1) + sch, _, _, _ := createSchedule(make(chan time.Time), sender) + + _ = sch.stateManager.ProcessEvalResults(context.Background(), sch.clock.Now(), rule, genEvalResults(sch.clock.Now()), nil, nil) require.NotEmpty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID)) factory := ruleFactoryFromScheduler(sch) @@ -519,6 +565,7 @@ func TestRuleRoutine(t *testing.T) { require.NoError(t, err) require.Empty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID)) + sender.AlertsSenderMock.AssertExpectations(t) }) }) diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 3194527626b..b503f7ad25c 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -242,7 +242,7 @@ func (st *Manager) DeleteStateByRuleUID(ctx context.Context, ruleKey ngModels.Al logger := st.log.FromContext(ctx) logger.Debug("Resetting state of the rule") - states := st.cache.removeByRuleUID(ruleKey.OrgID, ruleKey.UID) + states := st.ForgetStateByRuleUID(ctx, ruleKey) if len(states) == 0 { return nil @@ -280,11 +280,19 @@ func (st *Manager) DeleteStateByRuleUID(ctx context.Context, ruleKey ngModels.Al logger.Error("Failed to delete states that belong to a rule from database", "error", err) } } + logger.Info("Rules state was reset", "states", len(states)) return transitions } +func (st *Manager) ForgetStateByRuleUID(ctx context.Context, ruleKey ngModels.AlertRuleKeyWithGroup) []*State { + logger := st.log.FromContext(ctx) + logger.Debug("Removing rule state from cache") + + return st.cache.removeByRuleUID(ruleKey.OrgID, ruleKey.UID) +} + // ResetStateByRuleUID removes the rule instances from cache and instanceStore and saves state history. If the state // history has to be saved, rule must not be nil. func (st *Manager) ResetStateByRuleUID(ctx context.Context, rule *ngModels.AlertRule, reason string) []StateTransition { diff --git a/pkg/services/provisioning/alerting/rules_provisioner.go b/pkg/services/provisioning/alerting/rules_provisioner.go index 4baefedfebf..f41e48a94da 100644 --- a/pkg/services/provisioning/alerting/rules_provisioner.go +++ b/pkg/services/provisioning/alerting/rules_provisioner.go @@ -45,7 +45,8 @@ func (prov *defaultAlertRuleProvisioner) Provision(ctx context.Context, files []*AlertingFile) error { for _, file := range files { for _, group := range file.Groups { - u := provisionerUser(group.OrgID) + ctx, u := identity.WithServiceIdentitiy(ctx, group.OrgID) + folderUID, err := prov.getOrCreateFolderFullpath(ctx, group.FolderFullpath, group.OrgID) if err != nil { prov.logger.Error("failed to get or create folder", "folder", group.FolderFullpath, "org", group.OrgID, "err", err) @@ -120,11 +121,13 @@ func (prov *defaultAlertRuleProvisioner) getOrCreateFolderFullpath( func (prov *defaultAlertRuleProvisioner) getOrCreateFolderByTitle( ctx context.Context, folderName string, orgID int64, parentUID *string) (string, error) { + ctx, user := identity.WithServiceIdentitiy(ctx, orgID) + cmd := &folder.GetFolderQuery{ Title: &folderName, ParentUID: parentUID, OrgID: orgID, - SignedInUser: provisionerUser(orgID), + SignedInUser: user, } cmdResult, err := prov.folderService.Get(ctx, cmd) diff --git a/pkg/services/provisioning/dashboards/file_reader.go b/pkg/services/provisioning/dashboards/file_reader.go index e36bedda79e..b7c54ddcdb0 100644 --- a/pkg/services/provisioning/dashboards/file_reader.go +++ b/pkg/services/provisioning/dashboards/file_reader.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/metrics" @@ -147,6 +148,8 @@ func (fr *FileReader) isDatabaseAccessRestricted() bool { // storeDashboardsInFolder saves dashboards from the filesystem on disk to the folder from config func (fr *FileReader) storeDashboardsInFolder(ctx context.Context, filesFoundOnDisk map[string]os.FileInfo, dashboardRefs map[string]*dashboards.DashboardProvisioning, usageTracker *usageTracker) error { + ctx, _ = identity.WithServiceIdentitiy(ctx, fr.Cfg.OrgID) + folderID, folderUID, err := fr.getOrCreateFolder(ctx, fr.Cfg, fr.dashboardProvisioningService, fr.Cfg.Folder) if err != nil && !errors.Is(err, ErrFolderNameMissing) { return fmt.Errorf("%w with name %q: %w", ErrGetOrCreateFolder, fr.Cfg.Folder, err) @@ -177,6 +180,7 @@ func (fr *FileReader) storeDashboardsInFoldersFromFileStructure(ctx context.Cont folderName = filepath.Base(dashboardsFolder) } + ctx, _ = identity.WithServiceIdentitiy(ctx, fr.Cfg.OrgID) folderID, folderUID, err := fr.getOrCreateFolder(ctx, fr.Cfg, fr.dashboardProvisioningService, folderName) if err != nil && !errors.Is(err, ErrFolderNameMissing) { return fmt.Errorf("%w with name %q from file system structure: %w", ErrGetOrCreateFolder, folderName, err) @@ -342,38 +346,42 @@ func (fr *FileReader) getOrCreateFolder(ctx context.Context, cfg *config, servic return 0, "", ErrFolderNameMissing } - // TODO use folder service instead + user, err := identity.GetRequester(ctx) + if err != nil { + return 0, "", err + } + metrics.MFolderIDsServiceCount.WithLabelValues(metrics.Provisioning).Inc() - cmd := &dashboards.GetDashboardQuery{ - FolderID: util.Pointer(int64(0)), // nolint:staticcheck - OrgID: cfg.OrgID, + cmd := &folder.GetFolderQuery{ + OrgID: cfg.OrgID, + SignedInUser: user, } if cfg.FolderUID != "" { - cmd.UID = cfg.FolderUID + cmd.UID = &cfg.FolderUID } else { // provisioning depends on unique names //nolint:staticcheck cmd.Title = &folderName } - result, err := fr.dashboardStore.GetDashboard(ctx, cmd) - - if err != nil && !errors.Is(err, dashboards.ErrDashboardNotFound) { + result, err := fr.folderService.Get(ctx, cmd) + if err != nil && !errors.Is(err, dashboards.ErrFolderNotFound) { return 0, "", err } - // dashboard folder not found. create one. - if errors.Is(err, dashboards.ErrDashboardNotFound) { - // set dashboard folderUid if given - if cfg.FolderUID == accesscontrol.GeneralFolderUID { - return 0, "", dashboards.ErrFolderInvalidUID - } + // do not allow the creation of folder with uid "general" + if result != nil && result.UID == accesscontrol.GeneralFolderUID { + return 0, "", dashboards.ErrFolderInvalidUID + } + // dashboard folder not found. create one. + if errors.Is(err, dashboards.ErrFolderNotFound) { createCmd := &folder.CreateFolderCommand{ - OrgID: cfg.OrgID, - UID: cfg.FolderUID, - Title: folderName, + OrgID: cfg.OrgID, + UID: cfg.FolderUID, + Title: folderName, + SignedInUser: user, } f, err := service.SaveFolderForProvisionedDashboards(ctx, createCmd) @@ -385,10 +393,7 @@ func (fr *FileReader) getOrCreateFolder(ctx context.Context, cfg *config, servic return f.ID, f.UID, nil } - if !result.IsFolder { - return 0, "", fmt.Errorf("got invalid response. expected folder, found dashboard") - } - + //nolint:staticcheck return result.ID, result.UID, nil } diff --git a/pkg/services/provisioning/dashboards/file_reader_test.go b/pkg/services/provisioning/dashboards/file_reader_test.go index ac2741c86cc..52e9ac19e42 100644 --- a/pkg/services/provisioning/dashboards/file_reader_test.go +++ b/pkg/services/provisioning/dashboards/file_reader_test.go @@ -12,9 +12,20 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/dashboards/database" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" + "github.com/grafana/grafana/pkg/services/folder/folderimpl" + "github.com/grafana/grafana/pkg/services/supportbundles/supportbundlestest" + "github.com/grafana/grafana/pkg/services/tag/tagimpl" + "github.com/grafana/grafana/pkg/tests/testsuite" "github.com/grafana/grafana/pkg/util" ) @@ -28,6 +39,10 @@ const ( configName = "default" ) +func TestMain(m *testing.M) { + testsuite.Run(m) +} + func TestCreatingNewDashboardFileReader(t *testing.T) { setup := func() *config { return &config{ @@ -107,6 +122,17 @@ func TestDashboardFileReader(t *testing.T) { } } + sql, cfgT := db.InitTestDBWithCfg(t) + features := featuremgmt.WithFeatures(featuremgmt.FlagNestedFolders) + fStore := folderimpl.ProvideStore(sql) + tagService := tagimpl.ProvideService(sql) + dashStore, err := database.ProvideDashboardStore(sql, cfgT, features, tagService) + require.NoError(t, err) + folderStore := folderimpl.ProvideDashboardFolderStore(sql) + folderSvc := folderimpl.ProvideService(fStore, actest.FakeAccessControl{}, bus.ProvideBus(tracing.InitializeTracerForTest()), + dashStore, folderStore, nil, sql, featuremgmt.WithFeatures(), + supportbundlestest.NewFakeBundleService(), nil, cfgT, nil, tracing.InitializeTracerForTest()) + t.Run("Reading dashboards from disk", func(t *testing.T) { t.Run("Can read default dashboard", func(t *testing.T) { setup() @@ -116,8 +142,7 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("GetProvisionedDashboardData", mock.Anything, configName).Return(nil, nil).Once() fakeService.On("SaveFolderForProvisionedDashboards", mock.Anything, mock.Anything).Return(&folder.Folder{ID: 1}, nil).Once() fakeService.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{ID: 2}, nil).Times(2) - - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -137,7 +162,7 @@ func TestDashboardFileReader(t *testing.T) { inserted++ }) - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -174,7 +199,7 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("GetProvisionedDashboardData", mock.Anything, configName).Return(provisionedDashboard, nil).Once() - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -202,7 +227,7 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("GetProvisionedDashboardData", mock.Anything, configName).Return(provisionedDashboard, nil).Once() fakeService.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil).Once() - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -237,7 +262,7 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("GetProvisionedDashboardData", mock.Anything, configName).Return(provisionedDashboard, nil).Once() - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -265,7 +290,7 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("GetProvisionedDashboardData", mock.Anything, configName).Return(provisionedDashboard, nil).Once() fakeService.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil).Once() - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -280,7 +305,7 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("GetProvisionedDashboardData", mock.Anything, configName).Return(nil, nil).Once() fakeService.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil).Once() - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -297,7 +322,7 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("SaveFolderForProvisionedDashboards", mock.Anything, mock.Anything).Return(&folder.Folder{}, nil).Times(2) fakeService.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil).Times(3) - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -314,7 +339,7 @@ func TestDashboardFileReader(t *testing.T) { Folder: "", } - _, err := NewDashboardFileReader(cfg, logger, nil, nil, nil) + _, err := NewDashboardFileReader(cfg, logger, nil, nil, folderSvc) require.NotNil(t, err) }) @@ -322,7 +347,7 @@ func TestDashboardFileReader(t *testing.T) { setup() cfg.Options["path"] = brokenDashboards - _, err := NewDashboardFileReader(cfg, logger, nil, nil, nil) + _, err := NewDashboardFileReader(cfg, logger, nil, nil, folderSvc) require.NoError(t, err) }) @@ -335,14 +360,14 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("SaveFolderForProvisionedDashboards", mock.Anything, mock.Anything).Return(&folder.Folder{}, nil).Times(2) fakeService.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil).Times(2) - reader1, err := NewDashboardFileReader(cfg1, logger, nil, fakeStore, nil) + reader1, err := NewDashboardFileReader(cfg1, logger, nil, fakeStore, folderSvc) reader1.dashboardProvisioningService = fakeService require.NoError(t, err) err = reader1.walkDisk(context.Background()) require.NoError(t, err) - reader2, err := NewDashboardFileReader(cfg2, logger, nil, fakeStore, nil) + reader2, err := NewDashboardFileReader(cfg2, logger, nil, fakeStore, folderSvc) reader2.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -362,7 +387,7 @@ func TestDashboardFileReader(t *testing.T) { "folder": defaultDashboards, }, } - r, err := NewDashboardFileReader(cfg, logger, nil, nil, nil) + r, err := NewDashboardFileReader(cfg, logger, nil, nil, folderSvc) require.NoError(t, err) _, _, err = r.getOrCreateFolder(context.Background(), cfg, fakeService, cfg.Folder) @@ -382,10 +407,12 @@ func TestDashboardFileReader(t *testing.T) { } fakeService.On("SaveFolderForProvisionedDashboards", mock.Anything, mock.Anything).Return(&folder.Folder{ID: 1}, nil).Once() - r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) require.NoError(t, err) - _, _, err = r.getOrCreateFolder(context.Background(), cfg, fakeService, cfg.Folder) + ctx := context.Background() + ctx, _ = identity.WithServiceIdentitiy(ctx, 1) + _, _, err = r.getOrCreateFolder(ctx, cfg, fakeService, cfg.Folder) require.NoError(t, err) }) @@ -402,10 +429,12 @@ func TestDashboardFileReader(t *testing.T) { }, } - r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) require.NoError(t, err) - _, _, err = r.getOrCreateFolder(context.Background(), cfg, fakeService, cfg.Folder) + ctx := context.Background() + ctx, _ = identity.WithServiceIdentitiy(ctx, 1) + _, _, err = r.getOrCreateFolder(ctx, cfg, fakeService, cfg.Folder) require.ErrorIs(t, err, dashboards.ErrFolderInvalidUID) }) @@ -457,7 +486,7 @@ func TestDashboardFileReader(t *testing.T) { cfg.DisableDeletion = true - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) require.NoError(t, err) reader.dashboardProvisioningService = fakeService @@ -472,7 +501,7 @@ func TestDashboardFileReader(t *testing.T) { fakeService.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil).Once() fakeService.On("DeleteProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() - reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + reader, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) reader.dashboardProvisioningService = fakeService require.NoError(t, err) diff --git a/pkg/services/provisioning/dashboards/validator_test.go b/pkg/services/provisioning/dashboards/validator_test.go index de8d438d02f..788eb7e017a 100644 --- a/pkg/services/provisioning/dashboards/validator_test.go +++ b/pkg/services/provisioning/dashboards/validator_test.go @@ -8,9 +8,19 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/services/accesscontrol/actest" "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/dashboards/database" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" + "github.com/grafana/grafana/pkg/services/folder/folderimpl" + "github.com/grafana/grafana/pkg/services/supportbundles/supportbundlestest" + "github.com/grafana/grafana/pkg/services/tag/tagimpl" ) const ( @@ -31,16 +41,30 @@ func TestDuplicatesValidator(t *testing.T) { } logger := log.New("test.logger") + sql, cfgT := db.InitTestDBWithCfg(t) + features := featuremgmt.WithFeatures(featuremgmt.FlagNestedFolders) + fStore := folderimpl.ProvideStore(sql) + tagService := tagimpl.ProvideService(sql) + dashStore, err := database.ProvideDashboardStore(sql, cfgT, features, tagService) + require.NoError(t, err) + folderStore := folderimpl.ProvideDashboardFolderStore(sql) + folderSvc := folderimpl.ProvideService(fStore, actest.FakeAccessControl{}, bus.ProvideBus(tracing.InitializeTracerForTest()), + dashStore, folderStore, nil, sql, featuremgmt.WithFeatures(), + supportbundlestest.NewFakeBundleService(), nil, cfgT, nil, tracing.InitializeTracerForTest()) + t.Run("Duplicates validator should collect info about duplicate UIDs and titles within folders", func(t *testing.T) { const folderName = "duplicates-validator-folder" + ctx := context.Background() + ctx, _ = identity.WithServiceIdentitiy(ctx, 1) + fakeStore := &fakeDashboardStore{} - r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) require.NoError(t, err) fakeService.On("SaveFolderForProvisionedDashboards", mock.Anything, mock.Anything).Return(&folder.Folder{}, nil).Times(6) fakeService.On("GetProvisionedDashboardData", mock.Anything, mock.AnythingOfType("string")).Return([]*dashboards.DashboardProvisioning{}, nil).Times(4) fakeService.On("SaveProvisionedDashboard", mock.Anything, mock.Anything, mock.Anything).Return(&dashboards.Dashboard{}, nil).Times(5) - _, folderUID, err := r.getOrCreateFolder(context.Background(), cfg, fakeService, folderName) + _, folderUID, err := r.getOrCreateFolder(ctx, cfg, fakeService, folderName) require.NoError(t, err) identity := dashboardIdentity{folderUID: folderUID, title: "Grafana"} @@ -54,11 +78,11 @@ func TestDuplicatesValidator(t *testing.T) { Options: map[string]any{"path": dashboardContainingUID}, } - reader1, err := NewDashboardFileReader(cfg1, logger, nil, fakeStore, nil) + reader1, err := NewDashboardFileReader(cfg1, logger, nil, fakeStore, folderSvc) reader1.dashboardProvisioningService = fakeService require.NoError(t, err) - reader2, err := NewDashboardFileReader(cfg2, logger, nil, fakeStore, nil) + reader2, err := NewDashboardFileReader(cfg2, logger, nil, fakeStore, folderSvc) reader2.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -90,10 +114,13 @@ func TestDuplicatesValidator(t *testing.T) { t.Run("Duplicates validator should not collect info about duplicate UIDs and titles within folders for different orgs", func(t *testing.T) { const folderName = "duplicates-validator-folder" + ctx := context.Background() + ctx, _ = identity.WithServiceIdentitiy(ctx, 1) + fakeStore := &fakeDashboardStore{} - r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) require.NoError(t, err) - _, folderUID, err := r.getOrCreateFolder(context.Background(), cfg, fakeService, folderName) + _, folderUID, err := r.getOrCreateFolder(ctx, cfg, fakeService, folderName) require.NoError(t, err) identity := dashboardIdentity{folderUID: folderUID, title: "Grafana"} @@ -107,11 +134,11 @@ func TestDuplicatesValidator(t *testing.T) { Options: map[string]any{"path": dashboardContainingUID}, } - reader1, err := NewDashboardFileReader(cfg1, logger, nil, fakeStore, nil) + reader1, err := NewDashboardFileReader(cfg1, logger, nil, fakeStore, folderSvc) reader1.dashboardProvisioningService = fakeService require.NoError(t, err) - reader2, err := NewDashboardFileReader(cfg2, logger, nil, fakeStore, nil) + reader2, err := NewDashboardFileReader(cfg2, logger, nil, fakeStore, folderSvc) reader2.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -168,15 +195,15 @@ func TestDuplicatesValidator(t *testing.T) { Name: "third", Type: "file", OrgID: 2, Folder: "duplicates-validator-folder", Options: map[string]any{"path": twoDashboardsWithUID}, } - reader1, err := NewDashboardFileReader(cfg1, logger, nil, fakeStore, nil) + reader1, err := NewDashboardFileReader(cfg1, logger, nil, fakeStore, folderSvc) reader1.dashboardProvisioningService = fakeService require.NoError(t, err) - reader2, err := NewDashboardFileReader(cfg2, logger, nil, fakeStore, nil) + reader2, err := NewDashboardFileReader(cfg2, logger, nil, fakeStore, folderSvc) reader2.dashboardProvisioningService = fakeService require.NoError(t, err) - reader3, err := NewDashboardFileReader(cfg3, logger, nil, fakeStore, nil) + reader3, err := NewDashboardFileReader(cfg3, logger, nil, fakeStore, folderSvc) reader3.dashboardProvisioningService = fakeService require.NoError(t, err) @@ -193,9 +220,12 @@ func TestDuplicatesValidator(t *testing.T) { duplicates := duplicateValidator.getDuplicates() - r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, nil) + ctx := context.Background() + ctx, _ = identity.WithServiceIdentitiy(ctx, 1) + + r, err := NewDashboardFileReader(cfg, logger, nil, fakeStore, folderSvc) require.NoError(t, err) - _, folderUID, err := r.getOrCreateFolder(context.Background(), cfg, fakeService, cfg1.Folder) + _, folderUID, err := r.getOrCreateFolder(ctx, cfg, fakeService, cfg1.Folder) require.NoError(t, err) identity := dashboardIdentity{folderUID: folderUID, title: "Grafana"} @@ -210,9 +240,9 @@ func TestDuplicatesValidator(t *testing.T) { sort.Strings(titleUsageReaders) require.Equal(t, []string{"first"}, titleUsageReaders) - r, err = NewDashboardFileReader(cfg3, logger, nil, fakeStore, nil) + r, err = NewDashboardFileReader(cfg3, logger, nil, fakeStore, folderSvc) require.NoError(t, err) - _, folderUID, err = r.getOrCreateFolder(context.Background(), cfg3, fakeService, cfg3.Folder) + _, folderUID, err = r.getOrCreateFolder(ctx, cfg3, fakeService, cfg3.Folder) require.NoError(t, err) identity = dashboardIdentity{folderUID: folderUID, title: "Grafana"} diff --git a/pkg/services/provisioning/provisioning.go b/pkg/services/provisioning/provisioning.go index 451b3a098e3..fcc8a50c2f4 100644 --- a/pkg/services/provisioning/provisioning.go +++ b/pkg/services/provisioning/provisioning.go @@ -164,6 +164,7 @@ type ProvisioningServiceImpl struct { folderService folder.Service resourcePermissions accesscontrol.ReceiverPermissionsService tracer tracing.Tracer + onceInitProvisioners sync.Once } func (ps *ProvisioningServiceImpl) RunInitProvisioners(ctx context.Context) error { @@ -189,7 +190,19 @@ func (ps *ProvisioningServiceImpl) RunInitProvisioners(ctx context.Context) erro } func (ps *ProvisioningServiceImpl) Run(ctx context.Context) error { - err := ps.ProvisionDashboards(ctx) + var err error + + // run Init Provisioners only once + ps.onceInitProvisioners.Do(func() { + err = ps.RunInitProvisioners(ctx) + }) + + if err != nil { + // error already logged + return err + } + + err = ps.ProvisionDashboards(ctx) if err != nil { ps.log.Error("Failed to provision dashboard", "error", err) // Consider the allow list of errors for which running the provisioning service should not diff --git a/pkg/services/provisioning/provisioning_test.go b/pkg/services/provisioning/provisioning_test.go index de84250512a..beee1881a70 100644 --- a/pkg/services/provisioning/provisioning_test.go +++ b/pkg/services/provisioning/provisioning_test.go @@ -13,7 +13,11 @@ import ( dashboardstore "github.com/grafana/grafana/pkg/services/dashboards" "github.com/grafana/grafana/pkg/services/folder" "github.com/grafana/grafana/pkg/services/org" + "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginsettings" + "github.com/grafana/grafana/pkg/services/pluginsintegration/pluginstore" + prov_alerting "github.com/grafana/grafana/pkg/services/provisioning/alerting" "github.com/grafana/grafana/pkg/services/provisioning/dashboards" + "github.com/grafana/grafana/pkg/services/provisioning/datasources" "github.com/grafana/grafana/pkg/services/provisioning/utils" "github.com/grafana/grafana/pkg/services/searchV2" ) @@ -159,10 +163,17 @@ func setup(t *testing.T) *serviceTestStruct { serviceTest.dashboardProvisionerInstantiations++ return serviceTest.mock, nil }, - nil, - nil, + func(context.Context, string, datasources.BaseDataSourceService, datasources.CorrelationsStore, org.Service) error { + return nil + }, + func(context.Context, string, pluginstore.Store, pluginsettings.Service, org.Service) error { + return nil + }, searchStub, ) + service.provisionAlerting = func(context.Context, prov_alerting.ProvisionerConfig) error { + return nil + } serviceTest.service = service require.NoError(t, err) diff --git a/pkg/storage/unified/apistore/prepare.go b/pkg/storage/unified/apistore/prepare.go index 3e8275de1bf..b371444c66a 100644 --- a/pkg/storage/unified/apistore/prepare.go +++ b/pkg/storage/unified/apistore/prepare.go @@ -47,7 +47,7 @@ func (s *Storage) prepareObjectForStorage(ctx context.Context, newObject runtime return nil, err } if obj.GetName() == "" { - return nil, storage.ErrResourceVersionSetOnCreate + return nil, storage.NewInvalidObjError("", "missing name") } if obj.GetResourceVersion() != "" { return nil, storage.ErrResourceVersionSetOnCreate @@ -60,8 +60,11 @@ func (s *Storage) prepareObjectForStorage(ctx context.Context, newObject runtime // nolint:staticcheck id := obj.GetDeprecatedInternalID() if id < 1 { + // the ID must be smaller than 9007199254740991, otherwise we will lose prescision + // on the frontend, which uses the number type to store ids. The largest safe number in + // javascript is 9007199254740991, compared to 9223372036854775807 as the max int64 // nolint:staticcheck - obj.SetDeprecatedInternalID(s.snowflake.Generate().Int64()) + obj.SetDeprecatedInternalID(s.snowflake.Generate().Int64() & ((1 << 52) - 1)) } } diff --git a/pkg/storage/unified/apistore/prepare_test.go b/pkg/storage/unified/apistore/prepare_test.go new file mode 100644 index 00000000000..330c2791cfa --- /dev/null +++ b/pkg/storage/unified/apistore/prepare_test.go @@ -0,0 +1,140 @@ +package apistore + +import ( + "context" + "testing" + "time" + + "github.com/bwmarrin/snowflake" + "github.com/grafana/grafana/pkg/apimachinery/identity" + "github.com/grafana/grafana/pkg/apimachinery/utils" + "github.com/grafana/grafana/pkg/apis/dashboard/v0alpha1" + "github.com/grafana/grafana/pkg/services/user" + "github.com/stretchr/testify/require" + "golang.org/x/exp/rand" + "k8s.io/apimachinery/pkg/api/apitesting" + "k8s.io/apiserver/pkg/storage" +) + +func TestPrepareObjectForStorage(t *testing.T) { + _ = v0alpha1.AddToScheme(scheme) + node, err := snowflake.NewNode(rand.Int63n(1024)) + require.NoError(t, err) + s := &Storage{ + codec: apitesting.TestCodec(codecs, v0alpha1.DashboardResourceInfo.GroupVersion()), + snowflake: node, + opts: StorageOptions{ + LargeObjectSupport: nil, + }, + } + ctx := identity.WithRequester(context.Background(), &user.SignedInUser{UserID: 1, UserUID: "user-uid"}) + + t.Run("Error getting requester from context", func(t *testing.T) { + _, err := s.prepareObjectForStorage(context.Background(), nil) + require.Error(t, err) + require.Contains(t, err.Error(), "a Requester was not found in the context") + }) + + t.Run("Error on missing name", func(t *testing.T) { + dashboard := v0alpha1.Dashboard{} + _, err := s.prepareObjectForStorage(ctx, dashboard.DeepCopyObject()) + require.Error(t, err) + require.Contains(t, err.Error(), "missing name") + }) + + t.Run("Error on non-empty resource version", func(t *testing.T) { + dashboard := v0alpha1.Dashboard{} + dashboard.Name = "test-name" + dashboard.ResourceVersion = "123" + _, err := s.prepareObjectForStorage(ctx, dashboard.DeepCopyObject()) + require.Error(t, err) + require.Equal(t, storage.ErrResourceVersionSetOnCreate, err) + }) + + t.Run("Generate UID and leave deprecated ID empty, if not required", func(t *testing.T) { + dashboard := v0alpha1.Dashboard{} + dashboard.Name = "test-name" + + encodedData, err := s.prepareObjectForStorage(ctx, dashboard.DeepCopyObject()) + require.NoError(t, err) + + newObject, _, err := s.codec.Decode(encodedData, nil, &v0alpha1.Dashboard{}) + require.NoError(t, err) + obj, err := utils.MetaAccessor(newObject) + require.NoError(t, err) + require.NotEmpty(t, obj.GetUID(), "") + require.Empty(t, obj.GetDeprecatedInternalID()) // nolint:staticcheck + require.Empty(t, obj.GetGenerateName()) + require.Empty(t, obj.GetResourceVersion()) + require.Empty(t, obj.GetSelfLink()) + require.Empty(t, obj.GetUpdatedBy()) + require.Equal(t, obj.GetCreatedBy(), "user:user-uid") + updatedTS, err := obj.GetUpdatedTimestamp() + require.NoError(t, err) + require.Empty(t, updatedTS) + }) + + t.Run("Should keep repo info", func(t *testing.T) { + dashboard := v0alpha1.Dashboard{} + dashboard.Name = "test-name" + obj := dashboard.DeepCopyObject() + meta, err := utils.MetaAccessor(obj) + require.NoError(t, err) + now := time.Now() + meta.SetRepositoryInfo(&utils.ResourceRepositoryInfo{ + Name: "test-repo", + Path: "test/path", + Hash: "hash", + Timestamp: &now, + }) + + encodedData, err := s.prepareObjectForStorage(ctx, obj) + require.NoError(t, err) + + newObject, _, err := s.codec.Decode(encodedData, nil, &v0alpha1.Dashboard{}) + require.NoError(t, err) + meta, err = utils.MetaAccessor(newObject) + require.NoError(t, err) + require.Equal(t, meta.GetRepositoryHash(), "hash") + require.Equal(t, meta.GetRepositoryName(), "test-repo") + require.Equal(t, meta.GetRepositoryPath(), "test/path") + ts, err := meta.GetRepositoryTimestamp() + require.NoError(t, err) + parsed, err := time.Parse(time.RFC3339, now.UTC().Format(time.RFC3339)) + require.NoError(t, err) + require.Equal(t, ts, &parsed) + }) + + s.opts.RequireDeprecatedInternalID = true + t.Run("Should generate internal id", func(t *testing.T) { + dashboard := v0alpha1.Dashboard{} + dashboard.Name = "test-name" + + encodedData, err := s.prepareObjectForStorage(ctx, dashboard.DeepCopyObject()) + require.NoError(t, err) + newObject, _, err := s.codec.Decode(encodedData, nil, &v0alpha1.Dashboard{}) + require.NoError(t, err) + obj, err := utils.MetaAccessor(newObject) + require.NoError(t, err) + require.NotEmpty(t, obj.GetDeprecatedInternalID()) // nolint:staticcheck + // must be less than the max number value in javascript to avoid precision loss + require.LessOrEqual(t, obj.GetDeprecatedInternalID(), int64(9007199254740991)) // nolint:staticcheck + }) + + t.Run("Should use deprecated ID if given it", func(t *testing.T) { + dashboard := v0alpha1.Dashboard{} + dashboard.Name = "test-name" + obj := dashboard.DeepCopyObject() + meta, err := utils.MetaAccessor(obj) + require.NoError(t, err) + meta.SetDeprecatedInternalID(1) // nolint:staticcheck + + encodedData, err := s.prepareObjectForStorage(ctx, obj) + require.NoError(t, err) + newObject, _, err := s.codec.Decode(encodedData, nil, &v0alpha1.Dashboard{}) + require.NoError(t, err) + meta, err = utils.MetaAccessor(newObject) + require.NoError(t, err) + require.Equal(t, meta.GetDeprecatedInternalID(), int64(1)) // nolint:staticcheck + }) +} diff --git a/pkg/storage/unified/resource/go.mod b/pkg/storage/unified/resource/go.mod index 6f3f676f346..9126d4b0d3f 100644 --- a/pkg/storage/unified/resource/go.mod +++ b/pkg/storage/unified/resource/go.mod @@ -17,7 +17,6 @@ require ( github.com/grafana/grafana v11.4.0-00010101000000-000000000000+incompatible github.com/grafana/grafana-plugin-sdk-go v0.263.0 github.com/grafana/grafana/pkg/apimachinery v0.0.0-20250121113133-e747350fee2d - github.com/grafana/grafana/pkg/apiserver v0.0.0-20250121113133-e747350fee2d github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.2.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/prometheus/client_golang v1.20.5 @@ -122,6 +121,7 @@ require ( github.com/grafana/grafana-app-sdk/logging v0.29.0 // indirect github.com/grafana/grafana-aws-sdk v0.31.5 // indirect github.com/grafana/grafana-azure-sdk-go/v2 v2.1.6 // indirect + github.com/grafana/grafana/pkg/apiserver v0.0.0-20250121113133-e747350fee2d // indirect github.com/grafana/otel-profiling-go v0.5.1 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/grafana/sqlds/v4 v4.1.3 // indirect diff --git a/pkg/storage/unified/resource/metrics.go b/pkg/storage/unified/resource/metrics.go index 8747fa52a8a..f17d0133b2f 100644 --- a/pkg/storage/unified/resource/metrics.go +++ b/pkg/storage/unified/resource/metrics.go @@ -15,6 +15,7 @@ var ( type StorageApiMetrics struct { WatchEventLatency *prometheus.HistogramVec + PollerLatency prometheus.Histogram } func NewStorageMetrics() *StorageApiMetrics { @@ -29,6 +30,15 @@ func NewStorageMetrics() *StorageApiMetrics { NativeHistogramMaxBucketNumber: 160, NativeHistogramMinResetDuration: time.Hour, }, []string{"resource"}), + PollerLatency: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "storage_server", + Name: "poller_query_latency_seconds", + Help: "poller query latency", + Buckets: instrument.DefBuckets, + NativeHistogramBucketFactor: 1.1, // enable native histograms + NativeHistogramMaxBucketNumber: 160, + NativeHistogramMinResetDuration: time.Hour, + }), } }) @@ -37,8 +47,10 @@ func NewStorageMetrics() *StorageApiMetrics { func (s *StorageApiMetrics) Collect(ch chan<- prometheus.Metric) { s.WatchEventLatency.Collect(ch) + s.PollerLatency.Collect(ch) } func (s *StorageApiMetrics) Describe(ch chan<- *prometheus.Desc) { s.WatchEventLatency.Describe(ch) + s.PollerLatency.Describe(ch) } diff --git a/pkg/storage/unified/resource/search.go b/pkg/storage/unified/resource/search.go index a769cb37d2a..c7aa0c5d03a 100644 --- a/pkg/storage/unified/resource/search.go +++ b/pkg/storage/unified/resource/search.go @@ -230,6 +230,9 @@ func (s *searchSupport) CountRepositoryObjects(ctx context.Context, req *CountRe // Search implements ResourceIndexServer. func (s *searchSupport) Search(ctx context.Context, req *ResourceSearchRequest) (*ResourceSearchResponse, error) { + ctx, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Search") + defer span.End() + nsr := NamespacedResource{ Group: req.Options.Key.Group, Namespace: req.Options.Key.Namespace, @@ -394,10 +397,19 @@ func (s *searchSupport) init(ctx context.Context) error { // Async event func (s *searchSupport) handleEvent(ctx context.Context, evt *WrittenEvent) { + ctx, span := s.tracer.Start(ctx, tracingPrexfixSearch+"HandleEvent") if !slices.Contains([]WatchEvent_Type{WatchEvent_ADDED, WatchEvent_MODIFIED, WatchEvent_DELETED}, evt.Type) { s.log.Info("ignoring watch event", "type", evt.Type) return } + defer span.End() + span.SetAttributes( + attribute.String("event_type", evt.Type.String()), + attribute.String("namespace", evt.Key.Namespace), + attribute.String("group", evt.Key.Group), + attribute.String("resource", evt.Key.Resource), + attribute.String("name", evt.Key.Name), + ) nsr := NamespacedResource{ Namespace: evt.Key.Namespace, @@ -447,7 +459,9 @@ func (s *searchSupport) handleEvent(ctx context.Context, evt *WrittenEvent) { // record latency from when event was created to when it was indexed latencySeconds := float64(time.Now().UnixMicro()-evt.ResourceVersion) / 1e6 + span.AddEvent("index latency", trace.WithAttributes(attribute.Float64("latency_seconds", latencySeconds))) if latencySeconds > 5 { + s.log.Debug("high index latency object details", "resource", evt.Key.Resource, "latency_seconds", latencySeconds, "name", evt.Object.GetName(), "namespace", evt.Object.GetNamespace(), "uid", evt.Object.GetUID()) s.log.Warn("high index latency", "latency", latencySeconds) } if IndexMetrics != nil { @@ -460,6 +474,9 @@ func (s *searchSupport) getOrCreateIndex(ctx context.Context, key NamespacedReso return nil, fmt.Errorf("search is not configured properly (missing unifiedStorageSearch feature toggle?)") } + ctx, span := s.tracer.Start(ctx, tracingPrexfixSearch+"GetOrCreateIndex") + defer span.End() + // TODO??? // We want to block while building the index and return the same index for the key // simple mutex not great... we don't want to block while anything in building, just the same key diff --git a/pkg/storage/unified/resource/search_client.go b/pkg/storage/unified/resource/search_client.go index 43e0faa7b1e..a9c9c95ef9d 100644 --- a/pkg/storage/unified/resource/search_client.go +++ b/pkg/storage/unified/resource/search_client.go @@ -1,20 +1,24 @@ package resource import ( - "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/setting" ) +// Search Fallback was returning both Folders and Dashboards which resulted +// in issues with rendering the Folder UI. Also, filters are not implemented +// yet. For those reasons, we will be disabling Search Fallback for now func NewSearchClient(cfg *setting.Cfg, unifiedStorageConfigKey string, unifiedClient ResourceIndexClient, legacyClient ResourceIndexClient) ResourceIndexClient { - config, ok := cfg.UnifiedStorage[unifiedStorageConfigKey] - if !ok { - return legacyClient - } + // config, ok := cfg.UnifiedStorage[unifiedStorageConfigKey] + // if !ok { + // return legacyClient + // } - switch config.DualWriterMode { - case rest.Mode0, rest.Mode1, rest.Mode2: - return legacyClient - default: - return unifiedClient - } + // switch config.DualWriterMode { + // case rest.Mode0, rest.Mode1, rest.Mode2: + // return legacyClient + // default: + // return unifiedClient + // } + + return unifiedClient } diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go index d0c8b0f4ca7..145bd025cd7 100644 --- a/pkg/storage/unified/search/bleve.go +++ b/pkg/storage/unified/search/bleve.go @@ -177,6 +177,7 @@ func (b *bleveBackend) BuildIndex(ctx context.Context, fields: fields, standard: resource.StandardSearchFields(), features: b.features, + tracing: b.tracer, } idx.allFields, err = getAllFields(idx.standard, fields) @@ -250,6 +251,7 @@ type bleveIndex struct { batchSize int // ??? not totally sure the units here features featuremgmt.FeatureToggles + tracing trace.Tracer } // Write implements resource.DocumentIndex. @@ -404,6 +406,9 @@ func (b *bleveIndex) Search( req *resource.ResourceSearchRequest, federate []resource.ResourceIndex, // For federated queries, these will match the values in req.federate ) (*resource.ResourceSearchResponse, error) { + ctx, span := b.tracing.Start(ctx, tracingPrexfixBleve+"Search") + defer span.End() + if req.Options == nil || req.Options.Key == nil { return &resource.ResourceSearchResponse{ Error: resource.NewBadRequestError("missing query key"), @@ -418,7 +423,7 @@ func (b *bleveIndex) Search( } // Verifies the index federation - index, err := b.getIndex(req, federate) + index, err := b.getIndex(ctx, req, federate) if err != nil { return nil, err } @@ -448,7 +453,7 @@ func (b *bleveIndex) Search( response.QueryCost = float64(res.Cost) response.MaxScore = res.MaxScore - response.Results, err = b.hitsToTable(searchrequest.Fields, res.Hits, req.Explain) + response.Results, err = b.hitsToTable(ctx, searchrequest.Fields, res.Hits, req.Explain) if err != nil { return nil, err } @@ -465,6 +470,9 @@ func (b *bleveIndex) Search( } func (b *bleveIndex) DocCount(ctx context.Context, folder string) (int64, error) { + ctx, span := b.tracing.Start(ctx, tracingPrexfixBleve+"DocCount") + defer span.End() + if folder == "" { count, err := b.index.DocCount() return int64(count), err @@ -500,9 +508,13 @@ func (b *bleveIndex) verifyKey(key *resource.ResourceKey) *resource.ErrorResult } func (b *bleveIndex) getIndex( + ctx context.Context, req *resource.ResourceSearchRequest, federate []resource.ResourceIndex, ) (bleve.Index, error) { + _, span := b.tracing.Start(ctx, tracingPrexfixBleve+"getIndex") + defer span.End() + if len(req.Federated) != len(federate) { return nil, fmt.Errorf("federation is misconfigured") } @@ -527,6 +539,9 @@ func (b *bleveIndex) getIndex( } func (b *bleveIndex) toBleveSearchRequest(ctx context.Context, req *resource.ResourceSearchRequest, access authlib.AccessClient) (*bleve.SearchRequest, *resource.ErrorResult) { + ctx, span := b.tracing.Start(ctx, tracingPrexfixBleve+"toBleveSearchRequest") + defer span.End() + facets := bleve.FacetsRequest{} for _, f := range req.Facet { facets[f.Field] = bleve.NewFacetRequest(f.Field, int(f.Limit)) @@ -739,7 +754,10 @@ func requirementQuery(req *resource.Requirement, prefix string) (query.Query, *r ) } -func (b *bleveIndex) hitsToTable(selectFields []string, hits search.DocumentMatchCollection, explain bool) (*resource.ResourceTable, error) { +func (b *bleveIndex) hitsToTable(ctx context.Context, selectFields []string, hits search.DocumentMatchCollection, explain bool) (*resource.ResourceTable, error) { + _, span := b.tracing.Start(ctx, tracingPrexfixBleve+"hitsToTable") + defer span.End() + fields := []*resource.ResourceTableColumnDefinition{} for _, name := range selectFields { if name == "_all" { diff --git a/pkg/storage/unified/sql/backend.go b/pkg/storage/unified/sql/backend.go index 0401c77a25c..1690d129a77 100644 --- a/pkg/storage/unified/sql/backend.go +++ b/pkg/storage/unified/sql/backend.go @@ -805,6 +805,8 @@ func fetchLatestRV(ctx context.Context, x db.ContextExecer, d sqltemplate.Dialec func (b *backend) poll(ctx context.Context, grp string, res string, since int64, stream chan<- *resource.WrittenEvent) (int64, error) { ctx, span := b.tracer.Start(ctx, tracePrefix+"poll") defer span.End() + + start := time.Now() var records []*historyPollResponse err := b.db.WithTx(ctx, ReadCommittedRO, func(ctx context.Context, tx db.Tx) error { var err error @@ -820,6 +822,8 @@ func (b *backend) poll(ctx context.Context, grp string, res string, since int64, if err != nil { return 0, fmt.Errorf("poll history: %w", err) } + end := time.Now() + resource.NewStorageMetrics().PollerLatency.Observe(end.Sub(start).Seconds()) var nextRV int64 for _, rec := range records { @@ -847,6 +851,7 @@ func (b *backend) poll(ctx context.Context, grp string, res string, since int64, ResourceVersion: rec.ResourceVersion, // Timestamp: , // TODO: add timestamp } + b.log.Debug("poller sent event to stream", "namespace", rec.Key.Namespace, "group", rec.Key.Group, "resource", rec.Key.Resource, "name", rec.Key.Name, "action", rec.Action, "rv", rec.ResourceVersion) } return nextRV, nil diff --git a/openapi/README.md b/pkg/tests/apis/openapi_snapshots/README.md similarity index 100% rename from openapi/README.md rename to pkg/tests/apis/openapi_snapshots/README.md diff --git a/openapi/dashboard.grafana.app-v0alpha1.json b/pkg/tests/apis/openapi_snapshots/dashboard.grafana.app-v0alpha1.json similarity index 100% rename from openapi/dashboard.grafana.app-v0alpha1.json rename to pkg/tests/apis/openapi_snapshots/dashboard.grafana.app-v0alpha1.json diff --git a/openapi/folder.grafana.app-v0alpha1.json b/pkg/tests/apis/openapi_snapshots/folder.grafana.app-v0alpha1.json similarity index 100% rename from openapi/folder.grafana.app-v0alpha1.json rename to pkg/tests/apis/openapi_snapshots/folder.grafana.app-v0alpha1.json diff --git a/openapi/peakq.grafana.app-v0alpha1.json b/pkg/tests/apis/openapi_snapshots/peakq.grafana.app-v0alpha1.json similarity index 100% rename from openapi/peakq.grafana.app-v0alpha1.json rename to pkg/tests/apis/openapi_snapshots/peakq.grafana.app-v0alpha1.json diff --git a/openapi/provisioning.grafana.app-v0alpha1.json b/pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json similarity index 100% rename from openapi/provisioning.grafana.app-v0alpha1.json rename to pkg/tests/apis/openapi_snapshots/provisioning.grafana.app-v0alpha1.json diff --git a/pkg/tests/apis/core/openapi_test.go b/pkg/tests/apis/openapi_test.go similarity index 55% rename from pkg/tests/apis/core/openapi_test.go rename to pkg/tests/apis/openapi_test.go index f92c11fc7b3..f6db4aa6613 100644 --- a/pkg/tests/apis/core/openapi_test.go +++ b/pkg/tests/apis/openapi_test.go @@ -1,4 +1,4 @@ -package core +package apis import ( "bytes" @@ -17,7 +17,6 @@ import ( apimachineryversion "k8s.io/apimachinery/pkg/version" "github.com/grafana/grafana/pkg/services/featuremgmt" - "github.com/grafana/grafana/pkg/tests/apis" "github.com/grafana/grafana/pkg/tests/testinfra" "github.com/grafana/grafana/pkg/tests/testsuite" ) @@ -31,21 +30,7 @@ func TestIntegrationOpenAPIs(t *testing.T) { t.Skip("skipping integration test") } - check := []schema.GroupVersion{{ - Group: "dashboard.grafana.app", - Version: "v0alpha1", - }, { - Group: "folder.grafana.app", - Version: "v0alpha1", - }, { - Group: "provisioning.grafana.app", - Version: "v0alpha1", - }, { - Group: "peakq.grafana.app", - Version: "v0alpha1", - }} - - h := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{ + h := NewK8sTestHelper(t, testinfra.GrafanaOpts{ AppModeProduction: true, EnableFeatureToggles: []string{ featuremgmt.FlagKubernetesFoldersServiceV2, // Will be default on by G12 @@ -76,47 +61,67 @@ func TestIntegrationOpenAPIs(t *testing.T) { require.Equal(t, info.Minor, fmt.Sprintf("%d", v.Minor())) }) - t.Run("build open", func(t *testing.T) { - // Now write each OpenAPI spec to a static file - dir := filepath.Join("..", "..", "..", "..", "openapi") - for _, gv := range check { - path := fmt.Sprintf("/openapi/v3/apis/%s/%s", gv.Group, gv.Version) - rsp := apis.DoRequest(h, apis.RequestParams{ - Method: http.MethodGet, - Path: path, - User: h.Org1.Admin, - }, &apis.AnyResource{}) + dir := "openapi_snapshots" - require.NotNil(t, rsp.Response) - require.Equal(t, 200, rsp.Response.StatusCode, path) + for _, gv := range []schema.GroupVersion{{ + Group: "dashboard.grafana.app", + Version: "v0alpha1", + }, { + Group: "folder.grafana.app", + Version: "v0alpha1", + }, { + Group: "peakq.grafana.app", + Version: "v0alpha1", + }, { + Group: "provisioning.grafana.app", + Version: "v0alpha1", + }} { + VerifyOpenAPISnapshots(t, dir, gv, h) + } +} - var prettyJSON bytes.Buffer - err := json.Indent(&prettyJSON, rsp.Body, "", " ") - require.NoError(t, err) - pretty := prettyJSON.String() +// This function should be moved to oss (it is now a duplicate) +func VerifyOpenAPISnapshots(t *testing.T, dir string, gv schema.GroupVersion, h *K8sTestHelper) { + if gv.Group == "" { + return // skip invalid groups + } + path := fmt.Sprintf("/openapi/v3/apis/%s/%s", gv.Group, gv.Version) + t.Run(path, func(t *testing.T) { + rsp := DoRequest(h, RequestParams{ + Method: http.MethodGet, + Path: path, + User: h.Org1.Admin, + }, &AnyResource{}) - write := false - fpath := filepath.Join(dir, fmt.Sprintf("%s-%s.json", gv.Group, gv.Version)) + require.NotNil(t, rsp.Response) + require.Equal(t, 200, rsp.Response.StatusCode, path) - // nolint:gosec - // We can ignore the gosec G304 warning since this is a test and the function is only called with explicit paths - body, err := os.ReadFile(fpath) - if err == nil { - if !assert.JSONEq(t, string(body), pretty) { - t.Logf("openapi spec has changed: %s", path) - t.Fail() - write = true - } - } else { - t.Errorf("missing openapi spec for: %s", path) + var prettyJSON bytes.Buffer + err := json.Indent(&prettyJSON, rsp.Body, "", " ") + require.NoError(t, err) + pretty := prettyJSON.String() + + write := false + fpath := filepath.Join(dir, fmt.Sprintf("%s-%s.json", gv.Group, gv.Version)) + + // nolint:gosec + // We can ignore the gosec G304 warning since this is a test and the function is only called with explicit paths + body, err := os.ReadFile(fpath) + if err == nil { + if !assert.JSONEq(t, string(body), pretty) { + t.Logf("openapi spec has changed: %s", path) + t.Fail() write = true } + } else { + t.Errorf("missing openapi spec for: %s", path) + write = true + } - if write { - e2 := os.WriteFile(fpath, []byte(pretty), 0644) - if e2 != nil { - t.Errorf("error writing file: %s", e2.Error()) - } + if write { + e2 := os.WriteFile(fpath, []byte(pretty), 0644) + if e2 != nil { + t.Errorf("error writing file: %s", e2.Error()) } } }) diff --git a/yarn.lock b/yarn.lock index 53483ac0007..f6881a6c123 100644 --- a/yarn.lock +++ b/yarn.lock @@ -20982,8 +20982,8 @@ __metadata: linkType: hard "knip@npm:^5.10.0": - version: 5.43.5 - resolution: "knip@npm:5.43.5" + version: 5.43.6 + resolution: "knip@npm:5.43.6" dependencies: "@nodelib/fs.walk": "npm:3.0.1" "@snyk/github-codeowners": "npm:1.1.0" @@ -21007,7 +21007,7 @@ __metadata: bin: knip: bin/knip.js knip-bun: bin/knip-bun.js - checksum: 10/d6d7b6561b07fc1696aa109c822c4653c4cfdfd4aafb3bbb27e508415ff8b6e7d2fd14158cea800d59aa783e8fee25c641d1348374af86f743e2dee8140722d0 + checksum: 10/d843ed0f5b56baf5c29257308b0cf1956348cfa9d2b9b627420db023a6ccdaf54450f047fe900b263dc10291f395bc3eceef221dc6050e7fd55fb1fbe4fce3a2 languageName: node linkType: hard