rancher-docs/assets/js/d4d25003.476a4e1e.js

"use strict";(self.webpackChunkrancher_docs=self.webpackChunkrancher_docs||[]).push([[77224],{3905:(t,e,n)=>{n.d(e,{Zo:()=>s,kt:()=>p});var a=n(67294);function l(t,e,n){return e in t?Object.defineProperty(t,e,{value:n,enumerable:!0,configurable:!0,writable:!0}):t[e]=n,t}function r(t,e){var n=Object.keys(t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(t);e&&(a=a.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),n.push.apply(n,a)}return n}function d(t){for(var e=1;e<arguments.length;e++){var n=null!=arguments[e]?arguments[e]:{};e%2?r(Object(n),!0).forEach((function(e){l(t,e,n[e])})):Object.getOwnPropertyDescriptors?Object.defineProperties(t,Object.getOwnPropertyDescriptors(n)):r(Object(n)).forEach((function(e){Object.defineProperty(t,e,Object.getOwnPropertyDescriptor(n,e))}))}return t}function i(t,e){if(null==t)return{};var n,a,l=function(t,e){if(null==t)return{};var n,a,l={},r=Object.keys(t);for(a=0;a<r.length;a++)n=r[a],e.indexOf(n)>=0||(l[n]=t[n]);return l}(t,e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);for(a=0;a<r.length;a++)n=r[a],e.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(t,n)&&(l[n]=t[n])}return l}var o=a.createContext({}),m=function(t){var e=a.useContext(o),n=e;return t&&(n="function"==typeof t?t(e):d(d({},e),t)),n},s=function(t){var e=m(t.components);return a.createElement(o.Provider,{value:e},t.children)},k={inlineCode:"code",wrapper:function(t){var e=t.children;return a.createElement(a.Fragment,{},e)}},u=a.forwardRef((function(t,e){var n=t.components,l=t.mdxType,r=t.originalType,o=t.parentName,s=i(t,["components","mdxType","originalType","parentName"]),u=m(n),p=l,c=u["".concat(o,".").concat(p)]||u[p]||k[p]||r;return n?a.createElement(c,d(d({ref:e},s),{},{components:n})):a.createElement(c,d({ref:e},s))}));function p(t,e){var n=arguments,l=e&&e.mdxType;if("string"==typeof t||l){var r=n.length,d=new Array(r);d[0]=u;var i={};for(var o in e)hasOwnProperty.call(e,o)&&(i[o]=e[o]);i.originalType=t,i.mdxType="string"==typeof t?t:l,d[1]=i;for(var m=2;m<r;m++)d[m]=n[m];return a.createElement.apply(null,d)}return a.createElement.apply(null,n)}u.displayName="MDXCreateElement"},41006:(t,e,n)=>{n.r(e),n.d(e,{assets:()=>o,contentTitle:()=>d,default:()=>k,frontMatter:()=>r,metadata:()=>i,toc:()=>m});var a=n(87462),l=(n(67294),n(3905));const r={title:"PromQL Expression Reference"},d=void 0,i={unversionedId:"integrations-in-rancher/monitoring-and-alerting/promql-expressions",id:"version-2.6/integrations-in-rancher/monitoring-and-alerting/promql-expressions",title:"PromQL Expression Reference",description:"The PromQL expressions in this doc can be used to configure alerts.",source:"@site/versioned_docs/version-2.6/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md",sourceDirName:"integrations-in-rancher/monitoring-and-alerting",slug:"/integrations-in-rancher/monitoring-and-alerting/promql-expressions",permalink:"/v2.6/integrations-in-rancher/monitoring-and-alerting/promql-expressions",draft:!1,editUrl:"https://github.com/rancher/rancher-docs/edit/main/versioned_docs/version-2.6/integrations-in-rancher/monitoring-and-alerting/promql-expressions.md",tags:[],version:"2.6",lastUpdatedAt:1668452187,formattedLastUpdatedAt:"Nov 14, 2022",frontMatter:{title:"PromQL Expression Reference"},sidebar:"tutorialSidebar",previous:{title:"Windows Cluster Support for Monitoring V2",permalink:"/v2.6/integrations-in-rancher/monitoring-and-alerting/windows-support"},next:{title:"NeuVector Integration",permalink:"/v2.6/integrations-in-rancher/neuvector"}},o={},m=[{value:"Cluster Metrics",id:"cluster-metrics",level:2},{value:"Cluster CPU Utilization",id:"cluster-cpu-utilization",level:3},{value:"Cluster Load Average",id:"cluster-load-average",level:3},{value:"Cluster Memory Utilization",id:"cluster-memory-utilization",level:3},{value:"Cluster Disk Utilization",id:"cluster-disk-utilization",level:3},{value:"Cluster Disk I/O",id:"cluster-disk-io",level:3},{value:"Cluster Network Packets",id:"cluster-network-packets",level:3},{value:"Cluster Network I/O",id:"cluster-network-io",level:3},{value:"Node Metrics",id:"node-metrics",level:2},{value:"Node CPU Utilization",id:"node-cpu-utilization",level:3},{value:"Node Load Average",id:"node-load-average",level:3},{value:"Node Memory Utilization",id:"node-memory-utilization",level:3},{value:"Node Disk Utilization",id:"node-disk-utilization",level:3},{value:"Node Disk I/O",id:"node-disk-io",level:3},{value:"Node Network Packets",id:"node-network-packets",level:3},{value:"Node Network I/O",id:"node-network-io",level:3},{value:"Etcd Metrics",id:"etcd-metrics",level:2},{value:"Etcd Has a Leader",id:"etcd-has-a-leader",level:3},{value:"Number of Times the Leader Changes",id:"number-of-times-the-leader-changes",level:3},{value:"Number of Failed Proposals",id:"number-of-failed-proposals",level:3},{value:"GRPC Client Traffic",id:"grpc-client-traffic",level:3},{value:"Peer Traffic",id:"peer-traffic",level:3},{value:"DB Size",id:"db-size",level:3},{value:"Active Streams",id:"active-streams",level:3},{value:"Raft Proposals",id:"raft-proposals",level:3},{value:"RPC Rate",id:"rpc-rate",level:3},{value:"Disk Operations",id:"disk-operations",level:3},{value:"Disk Sync Duration",id:"disk-sync-duration",level:3},{value:"Kubernetes Components Metrics",id:"kubernetes-components-metrics",level:2},{value:"API Server Request Latency",id:"api-server-request-latency",level:3},{value:"API Server Request Rate",id:"api-server-request-rate",level:3},{value:"Scheduling Failed Pods",id:"scheduling-failed-pods",level:3},{value:"Controller Manager Queue Depth",id:"controller-manager-queue-depth",level:3},{value:"Scheduler E2E Scheduling Latency",id:"scheduler-e2e-scheduling-latency",level:3},{value:"Scheduler Preemption Attempts",id:"scheduler-preemption-attempts",level:3},{value:"Ingress Controller Connections",id:"ingress-controller-connections",level:3},{value:"Ingress Controller Request Process Time",id:"ingress-controller-request-process-time",level:3},{value:"Rancher Logging Metrics",id:"rancher-logging-metrics",level:2},{value:"Fluentd Buffer Queue Rate",id:"fluentd-buffer-queue-rate",level:3},{value:"Fluentd Input Rate",id:"fluentd-input-rate",level:3},{value:"Fluentd Output Errors Rate",id:"fluentd-output-errors-rate",level:3},{value:"Fluentd Output Rate",id:"fluentd-output-rate",level:3},{value:"Workload Metrics",id:"workload-metrics",level:2},{value:"Workload CPU Utilization",id:"workload-cpu-utilization",level:3},{value:"Workload Memory Utilization",id:"workload-memory-utilization",level:3},{value:"Workload Network Packets",id:"workload-network-packets",level:3},{value:"Workload Network I/O",id:"workload-network-io",level:3},{value:"Workload Disk I/O",id:"workload-disk-io",level:3},{value:"Pod Metrics",id:"pod-metrics",level:2},{value:"Pod CPU Utilization",id:"pod-cpu-utilization",level:3},{value:"Pod Memory Utilization",id:"pod-memory-utilization",level:3},{value:"Pod Network Packets",id:"pod-network-packets",level:3},{value:"Pod Network I/O",id:"pod-network-io",level:3},{value:"Pod Disk I/O",id:"pod-disk-io",level:3},{value:"Container Metrics",id:"container-metrics",level:2},{value:"Container CPU Utilization",id:"container-cpu-utilization",level:3},{value:"Container Memory Utilization",id:"container-memory-utilization",level:3},{value:"Container Disk I/O",id:"container-disk-io",level:3}],s={toc:m};function k(t){let{components:e,...n}=t;return(0,l.kt)("wrapper",(0,a.Z)({},s,n,{components:e,mdxType:"MDXLayout"}),(0,l.kt)("p",null,"The PromQL expressions in this doc can be used to configure alerts."),(0,l.kt)("p",null,"For more information about querying the Prometheus time series database, refer to the official ",(0,l.kt)("a",{parentName:"p",href:"https://prometheus.io/docs/prometheus/latest/querying/basics/"},"Prometheus documentation.")),(0,l.kt)("h2",{id:"cluster-metrics"},"Cluster Metrics"),(0,l.kt)("h3",{id:"cluster-cpu-utilization"},"Cluster CPU Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance))'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'1 - (avg(irate(node_cpu_seconds_total{mode="idle"}[5m])))'))))),(0,l.kt)("h3",{id:"cluster-load-average"},"Cluster Load Average"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"load1"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"load5"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"load15"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"}) by (instance)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"load1"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load1) by (instance) / count(node_cpu_seconds_total{mode="system"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"load5"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load5) by (instance) / count(node_cpu_seconds_total{mode="system"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"load15"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load15) by (instance) / count(node_cpu_seconds_total{mode="system"})')))))))),(0,l.kt)("h3",{id:"cluster-memory-utilization"},"Cluster Memory Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"1 - sum(node_memory_MemAvailable_bytes) by (instance) / sum(node_memory_MemTotal_bytes) by (instance)"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes)"))))),(0,l.kt)("h3",{id:"cluster-disk-utilization"},"Cluster Disk Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'(sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance) - sum(node_filesystem_free_bytes{device!="rootfs"}) by (instance)) / sum(node_filesystem_size_bytes{device!="rootfs"}) by (instance)'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'(sum(node_filesystem_size_bytes{device!="rootfs"}) - sum(node_filesystem_free_bytes{device!="rootfs"})) / sum(node_filesystem_size_bytes{device!="rootfs"})'))))),(0,l.kt)("h3",{id:"cluster-disk-io"},"Cluster Disk I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"read"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(node_disk_read_bytes_total[5m])) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"written"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(node_disk_written_bytes_total[5m])) by (instance)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"read"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(node_disk_read_bytes_total[5m]))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"written"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(node_disk_written_bytes_total[5m]))")))))))),(0,l.kt)("h3",{id:"cluster-network-packets"},"Cluster Network Packets"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-dropped"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_drop_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]",")) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-errs"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_errs_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]",")) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-packets"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_packets_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]",")) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-dropped"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_drop_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]",")) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-errs"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_errs_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]",")) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-packets"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_packets_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]",")) by (instance)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-dropped"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_drop_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-errs"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_errs_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-packets"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_packets_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-dropped"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_drop_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-errs"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_errs_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-packets"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_packets_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]","))")))))))),(0,l.kt)("h3",{id:"cluster-network-io"},"Cluster Network I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_bytes_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]",")) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_bytes_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]",")) by (instance)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_bytes_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_bytes_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'"}',"[5m]","))")))))))),(0,l.kt)("h2",{id:"node-metrics"},"Node Metrics"),(0,l.kt)("h3",{id:"node-cpu-utilization"},"Node CPU Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'avg(irate(node_cpu_seconds_total{mode!="idle", instance=~"$instance"}[5m])) by (mode)'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'1 - (avg(irate(node_cpu_seconds_total{mode="idle", instance=~"$instance"}[5m])))'))))),(0,l.kt)("h3",{id:"node-load-average"},"Node Load Average"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"load1"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"load5"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"load15"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"load1"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load1{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"load5"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load5{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"load15"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(node_load15{instance=~"$instance"}) / count(node_cpu_seconds_total{mode="system",instance=~"$instance"})')))))))),(0,l.kt)("h3",{id:"node-memory-utilization"},"Node Memory Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"})'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'1 - sum(node_memory_MemAvailable_bytes{instance=~"$instance"}) / sum(node_memory_MemTotal_bytes{instance=~"$instance"}) '))))),(0,l.kt)("h3",{id:"node-disk-utilization"},"Node Disk Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"}) by (device)) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) by (device)'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'(sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"}) - sum(node_filesystem_free_bytes{device!="rootfs",instance=~"$instance"})) / sum(node_filesystem_size_bytes{device!="rootfs",instance=~"$instance"})'))))),(0,l.kt)("h3",{id:"node-disk-io"},"Node Disk I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"read"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"written"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"read"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(node_disk_read_bytes_total{instance=~"$instance"}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"written"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(node_disk_written_bytes_total{instance=~"$instance"}[5m]))')))))))),(0,l.kt)("h3",{id:"node-network-packets"},"Node Network Packets"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-dropped"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_drop_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]",")) by (device)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-errs"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_errs_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]",")) by (device)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-packets"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_packets_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]",")) by (device)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-dropped"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_drop_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]",")) by (device)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-errs"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_errs_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]",")) by (device)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-packets"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_packets_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]",")) by (device)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-dropped"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_drop_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-errs"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_errs_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-packets"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_packets_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-dropped"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_drop_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-errs"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_errs_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-packets"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_packets_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]","))")))))))),(0,l.kt)("h3",{id:"node-network-io"},"Node Network I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_bytes_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]",")) by (device)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_bytes_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]",")) by (device)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_receive_bytes_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]","))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit"),(0,l.kt)("td",null,(0,l.kt)("code",null,'sum(rate(node_network_transmit_bytes_total{device!~"lo ',"|"," veth.","*"," ","|"," docker.","*"," ","|"," flannel.","*"," ","|"," cali.","*"," ","|"," cbr.","*",'",instance=~"$instance"}',"[5m]","))")))))))),(0,l.kt)("h2",{id:"etcd-metrics"},"Etcd Metrics"),(0,l.kt)("h3",{id:"etcd-has-a-leader"},"Etcd Has a Leader"),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"max(etcd_server_has_leader)")),(0,l.kt)("h3",{id:"number-of-times-the-leader-changes"},"Number of Times the Leader Changes"),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"max(etcd_server_leader_changes_seen_total)")),(0,l.kt)("h3",{id:"number-of-failed-proposals"},"Number of Failed Proposals"),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},"sum(etcd_server_proposals_failed_total)")),(0,l.kt)("h3",{id:"grpc-client-traffic"},"GRPC Client Traffic"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"in"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_network_client_grpc_received_bytes_total[5m])) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"out"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_network_client_grpc_sent_bytes_total[5m])) by (instance)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"in"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_network_client_grpc_received_bytes_total[5m]))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"out"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_network_client_grpc_sent_bytes_total[5m]))")))))))),(0,l.kt)("h3",{id:"peer-traffic"},"Peer Traffic"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"in"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_network_peer_received_bytes_total[5m])) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"out"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_network_peer_sent_bytes_total[5m])) by (instance)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"in"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_network_peer_received_bytes_total[5m]))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"out"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_network_peer_sent_bytes_total[5m]))")))))))),(0,l.kt)("h3",{id:"db-size"},"DB Size"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(etcd_debugging_mvcc_db_total_size_in_bytes) by (instance)"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(etcd_debugging_mvcc_db_total_size_in_bytes)"))))),(0,l.kt)("h3",{id:"active-streams"},"Active Streams"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"lease-watch"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"watch"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) by (instance)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"lease-watch"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(grpc_server_started_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Lease",grpc_type="bidi_stream"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"watch"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(grpc_server_started_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"}) - sum(grpc_server_handled_total{grpc_service="etcdserverpb.Watch",grpc_type="bidi_stream"})')))))))),(0,l.kt)("h3",{id:"raft-proposals"},"Raft Proposals"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"applied"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(increase(etcd_server_proposals_applied_total[5m])) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"committed"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(increase(etcd_server_proposals_committed_total[5m])) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"pending"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(increase(etcd_server_proposals_pending[5m])) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"failed"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(increase(etcd_server_proposals_failed_total[5m])) by (instance)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"applied"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(increase(etcd_server_proposals_applied_total[5m]))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"committed"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(increase(etcd_server_proposals_committed_total[5m]))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"pending"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(increase(etcd_server_proposals_pending[5m]))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"failed"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(increase(etcd_server_proposals_failed_total[5m]))")))))))),(0,l.kt)("h3",{id:"rpc-rate"},"RPC Rate"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"total"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(grpc_server_started_total{grpc_type="unary"}[5m])) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"fail"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m])) by (instance)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"total"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(grpc_server_started_total{grpc_type="unary"}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"fail"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(grpc_server_handled_total{grpc_type="unary",grpc_code!="OK"}[5m]))')))))))),(0,l.kt)("h3",{id:"disk-operations"},"Disk Operations"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"commit-called-by-backend"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m])) by (instance)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"fsync-called-by-wal"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m])) by (instance)")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"commit-called-by-backend"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_disk_backend_commit_duration_seconds_sum[1m]))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"fsync-called-by-wal"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(etcd_disk_wal_fsync_duration_seconds_sum[1m]))")))))))),(0,l.kt)("h3",{id:"disk-sync-duration"},"Disk Sync Duration"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"wal"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"db"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le))")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"wal"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) by (instance, le)))"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"db"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) by (instance, le)))")))))))),(0,l.kt)("h2",{id:"kubernetes-components-metrics"},"Kubernetes Components Metrics"),(0,l.kt)("h3",{id:"api-server-request-latency"},"API Server Request Latency"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance, verb) /1e+06"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"avg(apiserver_request_latencies_sum / apiserver_request_latencies_count) by (instance) /1e+06"))))),(0,l.kt)("h3",{id:"api-server-request-rate"},"API Server Request Rate"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(apiserver_request_count[5m])) by (instance, code)"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(apiserver_request_count[5m])) by (instance)"))))),(0,l.kt)("h3",{id:"scheduling-failed-pods"},"Scheduling Failed Pods"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(kube_pod_status_scheduled{condition="false"})'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(kube_pod_status_scheduled{condition="false"})'))))),(0,l.kt)("h3",{id:"controller-manager-queue-depth"},"Controller Manager Queue Depth"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"volumes"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(volumes_depth) by instance"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"deployment"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(deployment_depth) by instance"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"replicaset"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(replicaset_depth) by instance"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"service"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(service_depth) by instance"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"serviceaccount"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(serviceaccount_depth) by instance"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"endpoint"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(endpoint_depth) by instance"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"daemonset"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(daemonset_depth) by instance"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"statefulset"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(statefulset_depth) by instance"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"replicationmanager"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(replicationmanager_depth) by instance")))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"volumes"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(volumes_depth)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"deployment"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(deployment_depth)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"replicaset"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(replicaset_depth)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"service"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(service_depth)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"serviceaccount"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(serviceaccount_depth)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"endpoint"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(endpoint_depth)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"daemonset"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(daemonset_depth)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"statefulset"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(statefulset_depth)"))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"replicationmanager"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},"sum(replicationmanager_depth)")))))))),(0,l.kt)("h3",{id:"scheduler-e2e-scheduling-latency"},"Scheduler E2E Scheduling Latency"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(histogram_quantile(0.99, sum(scheduler_e2e_scheduling_latency_microseconds_bucket) by (le, instance)) / 1e+06)"))))),(0,l.kt)("h3",{id:"scheduler-preemption-attempts"},"Scheduler Preemption Attempts"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(scheduler_total_preemption_attempts[5m])) by (instance)"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(scheduler_total_preemption_attempts[5m]))"))))),(0,l.kt)("h3",{id:"ingress-controller-connections"},"Ingress Controller Connections"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"reading"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(nginx_ingress_controller_nginx_process_connections{state="reading"}) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"waiting"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(nginx_ingress_controller_nginx_process_connections{state="waiting"}) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"writing"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(nginx_ingress_controller_nginx_process_connections{state="writing"}) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"accepted"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m]))) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"active"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m]))) by (instance)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"handled"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m]))) by (instance)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"reading"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(nginx_ingress_controller_nginx_process_connections{state="reading"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"waiting"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(nginx_ingress_controller_nginx_process_connections{state="waiting"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"writing"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(nginx_ingress_controller_nginx_process_connections{state="writing"})'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"accepted"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="accepted"}[5m])))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"active"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="active"}[5m])))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"handled"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(ceil(increase(nginx_ingress_controller_nginx_process_connections_total{state="handled"}[5m])))')))))))),(0,l.kt)("h3",{id:"ingress-controller-request-process-time"},"Ingress Controller Request Process Time"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'topk(10, histogram_quantile(0.95,sum by (le, host, path)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'topk(10, histogram_quantile(0.95,sum by (le, host)(rate(nginx_ingress_controller_request_duration_seconds_bucket{host!="_"}[5m]))))'))))),(0,l.kt)("h2",{id:"rancher-logging-metrics"},"Rancher Logging Metrics"),(0,l.kt)("h3",{id:"fluentd-buffer-queue-rate"},"Fluentd Buffer Queue Rate"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(fluentd_output_status_buffer_queue_length[5m])) by (instance)"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(fluentd_output_status_buffer_queue_length[5m]))"))))),(0,l.kt)("h3",{id:"fluentd-input-rate"},"Fluentd Input Rate"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(fluentd_input_status_num_records_total[5m])) by (instance)"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(fluentd_input_status_num_records_total[5m]))"))))),(0,l.kt)("h3",{id:"fluentd-output-errors-rate"},"Fluentd Output Errors Rate"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(fluentd_output_status_num_errors[5m])) by (type)"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(fluentd_output_status_num_errors[5m]))"))))),(0,l.kt)("h3",{id:"fluentd-output-rate"},"Fluentd Output Rate"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(fluentd_output_status_num_records_total[5m])) by (instance)"))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},"sum(rate(fluentd_output_status_num_records_total[5m]))"))))),(0,l.kt)("h2",{id:"workload-metrics"},"Workload Metrics"),(0,l.kt)("h3",{id:"workload-cpu-utilization"},"Workload CPU Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"cfs throttled seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"user seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"system seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"usage seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"cfs throttled seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"user seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"system seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"usage seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))')))))))),(0,l.kt)("h3",{id:"workload-memory-utilization"},"Workload Memory Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""}) by (pod_name)'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(container_memory_working_set_bytes{namespace="$namespace",pod_name=~"$podName", container_name!=""})'))))),(0,l.kt)("h3",{id:"workload-network-packets"},"Workload Network Packets"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-packets"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-dropped"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-errors"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-packets"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-dropped"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-errors"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-packets"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-dropped"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-errors"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-packets"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-dropped"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-errors"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))')))))))),(0,l.kt)("h3",{id:"workload-network-io"},"Workload Network I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))')))))))),(0,l.kt)("h3",{id:"workload-disk-io"},"Workload Disk I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"read"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"write"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m])) by (pod_name)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"read"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"write"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name=~"$podName",container_name!=""}[5m]))')))))))),(0,l.kt)("h2",{id:"pod-metrics"},"Pod Metrics"),(0,l.kt)("h3",{id:"pod-cpu-utilization"},"Pod CPU Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"cfs throttled seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"usage seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"system seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"user seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m])) by (container_name)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"cfs throttled seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_cfs_throttled_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"usage seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_usage_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"system seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_system_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"user seconds"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_user_seconds_total{container_name!="POD",namespace="$namespace",pod_name="$podName", container_name!=""}[5m]))')))))))),(0,l.kt)("h3",{id:"pod-memory-utilization"},"Pod Memory Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""}) by (container_name)'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(container_memory_working_set_bytes{container_name!="POD",namespace="$namespace",pod_name="$podName",container_name!=""})'))))),(0,l.kt)("h3",{id:"pod-network-packets"},"Pod Network Packets"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-packets"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-dropped"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-errors"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-packets"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-dropped"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-errors"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-packets"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-dropped"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive-errors"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-packets"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_packets_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-dropped"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_packets_dropped_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit-errors"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_errors_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))')))))))),(0,l.kt)("h3",{id:"pod-network-io"},"Pod Network I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"receive"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_receive_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"transmit"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_network_transmit_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))')))))))),(0,l.kt)("h3",{id:"pod-disk-io"},"Pod Disk I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Detail"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"read"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"write"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m])) by (container_name)')))))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"Summary"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("table",null,(0,l.kt)("tr",null,(0,l.kt)("td",null,"read"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))'))),(0,l.kt)("tr",null,(0,l.kt)("td",null,"write"),(0,l.kt)("td",null,(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name!=""}[5m]))')))))))),(0,l.kt)("h2",{id:"container-metrics"},"Container Metrics"),(0,l.kt)("h3",{id:"container-cpu-utilization"},"Container CPU Utilization"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"cfs throttled seconds"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_cfs_throttled_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"usage seconds"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_usage_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"system seconds"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_system_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"user seconds"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_cpu_user_seconds_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))'))))),(0,l.kt)("h3",{id:"container-memory-utilization"},"Container Memory Utilization"),(0,l.kt)("p",null,(0,l.kt)("inlineCode",{parentName:"p"},'sum(container_memory_working_set_bytes{namespace="$namespace",pod_name="$podName",container_name="$containerName"})')),(0,l.kt)("h3",{id:"container-disk-io"},"Container Disk I/O"),(0,l.kt)("table",null,(0,l.kt)("thead",{parentName:"table"},(0,l.kt)("tr",{parentName:"thead"},(0,l.kt)("th",{parentName:"tr",align:null},"Catalog"),(0,l.kt)("th",{parentName:"tr",align:null},"Expression"))),(0,l.kt)("tbody",{parentName:"table"},(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"read"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_reads_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))'))),(0,l.kt)("tr",{parentName:"tbody"},(0,l.kt)("td",{parentName:"tr",align:null},"write"),(0,l.kt)("td",{parentName:"tr",align:null},(0,l.kt)("inlineCode",{parentName:"td"},'sum(rate(container_fs_writes_bytes_total{namespace="$namespace",pod_name="$podName",container_name="$containerName"}[5m]))'))))))}k.isMDXComponent=!0}}]);