catalog/incubator/mimir/0.2.0/output.yaml

---
# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-alertmanager
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: alertmanager
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: alertmanager
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/compactor/compactor-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-compactor
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: compactor
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: compactor
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/distributor/distributor-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-distributor
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: distributor
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: distributor
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/index-cache/index-cache-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-index-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: index-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: index-cache
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/ingester/ingester-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-ingester
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: ingester
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/metadata-cache/metadata-cache-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-metadata-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: metadata-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: metadata-cache
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/nginx/nginx-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-nginx
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: nginx
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: nginx
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/overrides-exporter/overrides-exporter-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-overrides-exporter
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: overrides-exporter
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: overrides-exporter
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/querier/querier-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-querier
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: querier
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: querier
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/query-frontend/query-frontend-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-query-frontend
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-frontend
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: query-frontend
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-query-scheduler
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-scheduler
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: query-scheduler
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/results-cache/results-cache-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-results-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: results-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: results-cache
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/ruler/ruler-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-ruler
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ruler
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: ruler
  maxUnavailable: 1
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: release-name-mimir-store-gateway
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: store-gateway
  maxUnavailable: 1
---
# Source: mimir/charts/kps/charts/kube-state-metrics/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:    
    helm.sh/chart: kube-state-metrics-5.16.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.10.1"
    release: release-name
  name: release-name-kube-state-metrics
  namespace: default
---
# Source: mimir/charts/kps/charts/prometheus-node-exporter/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: release-name-prometheus-node-exporter
  namespace: default
  labels:
    helm.sh/chart: prometheus-node-exporter-4.30.2
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: prometheus-node-exporter
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.7.0"
    jobLabel: node-exporter
    release: release-name
---
# Source: mimir/charts/mimir/charts/minio/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: "minio-sa"
---
# Source: mimir/charts/mimir/charts/rollout_operator/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: release-name-rollout-operator
  labels:
    helm.sh/chart: rollout-operator-0.10.0
    app.kubernetes.io/name: rollout-operator
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "v0.9.0"
    app.kubernetes.io/managed-by: Helm
---
# Source: mimir/charts/mimir/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: release-name-mimir
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
---
# Source: mimir/charts/mimir/charts/minio/templates/secrets.yaml
apiVersion: v1
kind: Secret
metadata:
  name: release-name-minio
  labels:
    app: minio
    chart: minio-5.0.14
    release: release-name
    heritage: Helm
type: Opaque
data:
  rootUser: "Z3JhZmFuYS1taW1pcg=="
  rootPassword: "c3VwZXJzZWNyZXQ="
---
# Source: mimir/charts/mimir/templates/metamonitoring/logs-instance-usernames-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: release-name-mimir-logs-instance-usernames
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: meta-monitoring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
data:
---
# Source: mimir/charts/mimir/templates/metamonitoring/metrics-instance-usernames-secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: release-name-mimir-metrics-instance-usernames
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: meta-monitoring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
data:
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/alertmanager-overview.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-alertmanager-overview
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  alertmanager-overview.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"current set of alerts stored in the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(alertmanager_alerts{namespace=~\"$namespace\",service=~\"$service\"}) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Alerts","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"none","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"none","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"rate of successful and invalid alerts received by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(alertmanager_alerts_received_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Received","refId":"A"},{"expr":"sum(rate(alertmanager_alerts_invalid_total{namespace=~\"$namespace\",service=~\"$service\"}[$__rate_interval])) by (namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Invalid","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Alerts receive rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Alerts","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"rate of successful and invalid notifications sent by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"integration","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(alertmanager_notifications_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Total","refId":"A"},{"expr":"sum(rate(alertmanager_notifications_failed_total{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (integration,namespace,service,instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Failed","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"$integration: Notifications Send Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"latency of notifications sent by the Alertmanager","fill":1,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":"integration","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99,\n  sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} 99th Percentile","refId":"A"},{"expr":"histogram_quantile(0.50,\n  sum(rate(alertmanager_notification_latency_seconds_bucket{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (le,namespace,service,instance)\n) \n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Median","refId":"B"},{"expr":"sum(rate(alertmanager_notification_latency_seconds_sum{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n/\nsum(rate(alertmanager_notification_latency_seconds_count{namespace=~\"$namespace\",service=~\"$service\", integration=\"$integration\"}[$__rate_interval])) by (namespace,service,instance)\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Average","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"$integration: Notification Duration","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Notifications","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["alertmanager-mixin"],"templating":{"list":[{"current":{"text":"Prometheus","value":"Prometheus"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":"namespace","multi":false,"name":"namespace","options":[],"query":"label_values(alertmanager_alerts, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":"service","multi":false,"name":"service","options":[],"query":"label_values(alertmanager_alerts, service)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"all","value":"$__all"},"datasource":"$datasource","hide":2,"includeAll":true,"label":null,"multi":false,"name":"integration","options":[],"query":"label_values(alertmanager_notifications_total{integration=~\".*\"}, integration)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Alertmanager / Overview","uid":"alertmanager-overview","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/apiserver.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-apiserver
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  apiserver.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"content":"The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.","datasource":null,"description":"The SLO (service level objective) and other metrics displayed on this dashboard are for informational purposes only.","gridPos":{"h":2,"w":24,"x":0,"y":0},"id":2,"mode":"markdown","span":12,"title":"Notice","type":"text"}],"refresh":"10s","rows":[{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":3,"description":"How many percent of requests (both read and write) in 30 days have been answered successfully and fast enough?","format":"percentunit","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":3,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":4,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Availability (30d) > 99.000%","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"avg"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":3,"description":"How much error budget is left looking at our 0.990% availability guarantees?","fill":10,"fillGradient":0,"gridPos":{},"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":8,"stack":false,"steppedLine":false,"targets":[{"expr":"100 * (apiserver_request:availability30d{verb=\"all\", cluster=\"$cluster\"} - 0.990000)","format":"time_series","intervalFactor":2,"legendFormat":"errorbudget","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ErrorBudget (30d) > 99.000%","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"decimals":3,"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"decimals":3,"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":3,"description":"How many percent of read requests (LIST,GET) in 30 days have been answered successfully and fast enough?","format":"percentunit","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":5,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"apiserver_request:availability30d{verb=\"read\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Read Availability (30d)","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"avg"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many read requests (LIST,GET) per second do the apiservers get by code?","fill":10,"fillGradient":0,"gridPos":{},"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/2../i","color":"#56A64B"},{"alias":"/3../i","color":"#F2CC0C"},{"alias":"/4../i","color":"#3274D9"},{"alias":"/5../i","color":"#E02F44"}],"spaceLength":10,"span":3,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{ code }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Read SLI - Requests","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many percent of read requests (LIST,GET) per second are returned with errors (5xx)?","fill":1,"fillGradient":0,"gridPos":{},"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"read\", cluster=\"$cluster\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{ resource }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Read SLI - Errors","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many seconds is the 99th percentile for reading (LIST|GET) a given resource?","fill":1,"fillGradient":0,"gridPos":{},"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile{verb=\"read\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{ resource }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Read SLI - Duration","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":3,"description":"How many percent of write requests (POST|PUT|PATCH|DELETE) in 30 days have been answered successfully and fast enough?","format":"percentunit","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":9,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"apiserver_request:availability30d{verb=\"write\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Write Availability (30d)","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"avg"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many write requests (POST|PUT|PATCH|DELETE) per second do the apiservers get by code?","fill":10,"fillGradient":0,"gridPos":{},"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/2../i","color":"#56A64B"},{"alias":"/3../i","color":"#F2CC0C"},{"alias":"/4../i","color":"#3274D9"},{"alias":"/5../i","color":"#E02F44"}],"spaceLength":10,"span":3,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (code) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{ code }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Write SLI - Requests","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many percent of write requests (POST|PUT|PATCH|DELETE) per second are returned with errors (5xx)?","fill":1,"fillGradient":0,"gridPos":{},"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\",code=~\"5..\", cluster=\"$cluster\"}) / sum by (resource) (code_resource:apiserver_request_total:rate5m{verb=\"write\", cluster=\"$cluster\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{ resource }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Write SLI - Errors","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"How many seconds is the 99th percentile for writing (POST|PUT|PATCH|DELETE) a given resource?","fill":1,"fillGradient":0,"gridPos":{},"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile{verb=\"write\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{ resource }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Write SLI - Duration","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Add Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":14,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Depth","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":15,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])) by (instance, name, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Latency","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":16,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":17,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}[$__rate_interval])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":18,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"go_goroutines{job=\"apiserver\",instance=~\"$instance\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Goroutines","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"apiserver\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":true,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(up{job=\"apiserver\", cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / API server","uid":"09ec8aa1e996d6ffcd6817bbaff4db1b","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/cluster-total.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-cluster-total
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  cluster-total.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":1},"id":3,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":1},"id":4,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"columns":[{"text":"Time","value":"Time"},{"text":"Value #A","value":"Value #A"},{"text":"Value #B","value":"Value #B"},{"text":"Value #C","value":"Value #C"},{"text":"Value #D","value":"Value #D"},{"text":"Value #E","value":"Value #E"},{"text":"Value #F","value":"Value #F"},{"text":"Value #G","value":"Value #G"},{"text":"Value #H","value":"Value #H"},{"text":"namespace","value":"namespace"}],"datasource":"$datasource","fill":1,"fontSize":"90%","gridPos":{"h":9,"w":24,"x":0,"y":10},"id":5,"lines":true,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null as zero","renderer":"flot","scroll":true,"showHeader":true,"sort":{"col":0,"desc":false},"spaceLength":10,"span":24,"styles":[{"alias":"Time","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Time","thresholds":[],"type":"hidden","unit":"short"},{"alias":"Current Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Average Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Average Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"pps"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTooltip":"Drill down","linkUrl":"d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"}],"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"A","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"B","step":10},{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"C","step":10},{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"D","step":10},{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"E","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"F","step":10},{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"G","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"H","step":10}],"timeFrom":null,"timeShift":null,"title":"Current Status","type":"table"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"id":6,"panels":[{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":11},"id":7,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":11},"id":8,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Bandwidth","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":11},"id":9,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth History","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":12},"id":10,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":21},"id":11,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":12,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":31},"id":13,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":40},"id":14,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":31},"id":15,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":50},"id":16,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":59},"id":17,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\".+\"}[$interval:$resolution])) by (namespace))","format":"time_series","intervalFactor":1,"legendFormat":"{{namespace}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":59},"id":18,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[{"targetBlank":true,"title":"What is TCP Retransmit?","url":"https://accedian.com/enterprises/blog/network-packet-loss-retransmissions-and-duplicate-acknowledgements/"}],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_OutSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of TCP Retransmits out of all sent segments","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":24,"x":0,"y":59},"id":19,"legend":{"alignAsTable":true,"avg":true,"current":true,"hideEmpty":true,"hideZero":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":2,"links":[{"targetBlank":true,"title":"Why monitor SYN retransmits?","url":"https://github.com/prometheus/node_exporter/issues/1023#issuecomment-408128365"}],"minSpan":24,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(rate(node_netstat_TcpExt_TCPSynRetrans{cluster=\"$cluster\"}[$interval:$resolution]) / rate(node_netstat_Tcp_RetransSegs{cluster=\"$cluster\"}[$interval:$resolution])) by (instance))","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of TCP SYN Retransmits out of all retransmits","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Cluster","uid":"ff635a025bcfea7bc3dd4f508990a3e9","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/controller-manager.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-controller-manager
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  controller-manager.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"10s","rows":[{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":2,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":2,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(up{cluster=\"$cluster\", job=\"kube-controller-manager\"})","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Up","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"min"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(workqueue_adds_total{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Add Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(workqueue_depth{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Depth","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, name, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} {{name}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Work Queue Latency","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"2xx","refId":"A"},{"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"3xx","refId":"B"},{"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"4xx","refId":"C"},{"expr":"sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"5xx","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Kube API Request Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":8,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Post Request Latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Get Request Latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"process_resident_memory_bytes{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}[$__rate_interval])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"go_goroutines{cluster=\"$cluster\", job=\"kube-controller-manager\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Goroutines","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-controller-manager\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":true,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(up{cluster=\"$cluster\", job=\"kube-controller-manager\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Controller Manager","uid":"72e0e05bef5099e5f049b05fdc429ed4","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/etcd.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-etcd
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  etcd.json: |-
    {"description":"etcd sample Grafana dashboard with Prometheus","panels":[{"datasource":{"type":"datasource","uid":"-- Mixed --"},"gridPos":{"h":7,"w":6,"x":0,"y":0},"id":1,"interval":"1m","options":{"colorMode":"none","graphMode":"none","reduceOptions":{"calcs":["lastNotNull"]}},"pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(etcd_server_has_leader{job=~\".*etcd.*\", job=\"$cluster\"})","legendFormat":"{{cluster}} - {{namespace}}\n"}],"title":"Up","type":"stat"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"ops"}},"gridPos":{"h":7,"w":10,"x":6,"y":0},"id":2,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(grpc_server_started_total{job=~\".*etcd.*\", job=\"$cluster\",grpc_type=\"unary\"}[$__rate_interval]))","legendFormat":"RPC rate"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(grpc_server_handled_total{job=~\".*etcd.*\", job=\"$cluster\",grpc_type=\"unary\",grpc_code=~\"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded\"}[$__rate_interval]))","legendFormat":"RPC failed rate"}],"title":"RPC rate","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":16,"y":0},"id":3,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(grpc_server_started_total{job=~\".*etcd.*\",job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})","legendFormat":"Watch streams"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(grpc_server_started_total{job=~\".*etcd.*\",job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})","legendFormat":"Lease streams"}],"title":"Active streams","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":0,"y":25},"id":4,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"etcd_mvcc_db_total_size_in_bytes{job=~\".*etcd.*\", job=\"$cluster\"}","legendFormat":"{{instance}} DB size"}],"title":"DB size","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"s"}},"gridPos":{"h":7,"w":8,"x":8,"y":25},"id":5,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance, le))","legendFormat":"{{instance}} WAL fsync"},{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance, le))","legendFormat":"{{instance}} DB fsync"}],"title":"Disk sync duration","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"bytes"}},"gridPos":{"h":7,"w":8,"x":16,"y":25},"id":6,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"process_resident_memory_bytes{job=~\".*etcd.*\", job=\"$cluster\"}","legendFormat":"{{instance}} resident memory"}],"title":"Memory","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":0,"y":50},"id":7,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"rate(etcd_network_client_grpc_received_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])","legendFormat":"{{instance}} client traffic in"}],"title":"Client traffic in","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":6,"y":50},"id":8,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"rate(etcd_network_client_grpc_sent_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])","legendFormat":"{{instance}} client traffic out"}],"title":"Client traffic out","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":12,"y":50},"id":9,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(etcd_network_peer_received_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance)","legendFormat":"{{instance}} peer traffic in"}],"title":"Peer traffic in","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"Bps"}},"gridPos":{"h":7,"w":6,"x":18,"y":50},"id":10,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"sum(rate(etcd_network_peer_sent_bytes_total{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])) by (instance)","legendFormat":"{{instance}} peer traffic out"}],"title":"Peer traffic out","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":0,"y":75},"id":11,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\", job=\"$cluster\"}[1d])","legendFormat":"{{instance}} total leader elections per day"}],"title":"Raft proposals","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"}}},"gridPos":{"h":7,"w":8,"x":8,"y":75},"id":12,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"changes(etcd_server_leader_changes_seen_total{job=~\".*etcd.*\", job=\"$cluster\"}[1d])","legendFormat":"{{instance}} total leader elections per day"}],"title":"Total leader elections per day","type":"timeseries"},{"datasource":{"type":"datasource","uid":"-- Mixed --"},"fieldConfig":{"defaults":{"custom":{"fillOpacity":0,"lineWidth":2,"showPoints":"never"},"unit":"s"}},"gridPos":{"h":7,"w":8,"x":16,"y":75},"id":13,"interval":"1m","pluginVersion":"v10.0.0","targets":[{"datasource":{"type":"prometheus","uid":"$datasource"},"expr":"histogram_quantile(0.99, sum by (instance, le) (rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~\".*etcd.*\", job=\"$cluster\"}[$__rate_interval])))","legendFormat":"{{instance}} peer round trip time"}],"title":"Peer round trip time","type":"timeseries"}],"refresh":"10s","schemaVersion":36,"tags":["etcd-mixin"],"templating":{"list":[{"label":"Data Source","name":"datasource","query":"prometheus","type":"datasource"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"label":"cluster","name":"cluster","query":"label_values(etcd_server_has_leader{job=~\".*etcd.*\"}, job)","refresh":2,"type":"query","hide":2}]},"time":{"from":"now-15m","to":"now"},"timezone": "utc","title":"etcd","uid":"c2f4e12cdf69feb95caa41a5a1b423d9"}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/grafana-overview.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-grafana-overview
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  grafana-overview.json: |-
    {"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"id":3085,"iteration":1631554945276,"links":[],"panels":[{"datasource":"$datasource","fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":6,"x":0,"y":0},"id":6,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.1.3","targets":[{"expr":"grafana_alerting_result_total{job=~\"$job\", instance=~\"$instance\", state=\"alerting\"}","instant":true,"interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Firing Alerts","type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":6,"x":6,"y":0},"id":8,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.1.3","targets":[{"expr":"sum(grafana_stat_totals_dashboard{job=~\"$job\", instance=~\"$instance\"})","interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Dashboards","type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{"align":null,"displayMode":"auto"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":12,"x":12,"y":0},"id":10,"options":{"showHeader":true},"pluginVersion":"8.1.3","targets":[{"expr":"grafana_build_info{job=~\"$job\", instance=~\"$instance\"}","instant":true,"interval":"","legendFormat":"","refId":"A"}],"timeFrom":null,"timeShift":null,"title":"Build Info","transformations":[{"id":"labelsToFields","options":{}},{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"branch":true,"container":true,"goversion":true,"namespace":true,"pod":true,"revision":true},"indexByName":{"Time":7,"Value":11,"branch":4,"container":8,"edition":2,"goversion":6,"instance":1,"job":0,"namespace":9,"pod":10,"revision":5,"version":3},"renameByName":{}}}],"type":"table"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":5},"hiddenSeries":false,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"8.1.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (status_code) (irate(grafana_http_request_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[1m])) ","interval":"","legendFormat":"{{status_code}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"RPS","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"$$hashKey":"object:157","format":"reqps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"$$hashKey":"object:158","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":5},"hiddenSeries":false,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"nullPointMode":"null","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"8.1.3","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"exemplar":true,"expr":"histogram_quantile(0.99, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1","interval":"","legendFormat":"99th Percentile","refId":"A"},{"exemplar":true,"expr":"histogram_quantile(0.50, sum(irate(grafana_http_request_duration_seconds_bucket{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) by (le)) * 1","interval":"","legendFormat":"50th Percentile","refId":"B"},{"exemplar":true,"expr":"sum(irate(grafana_http_request_duration_seconds_sum{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval])) * 1 / sum(irate(grafana_http_request_duration_seconds_count{instance=~\"$instance\", job=~\"$job\"}[$__rate_interval]))","interval":"","legendFormat":"Average","refId":"C"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Request Latency","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"$$hashKey":"object:210","format":"ms","label":null,"logBase":1,"max":null,"min":null,"show":true},{"$$hashKey":"object:211","format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}}],"schemaVersion":30,"style":"dark","tags":[],"templating":{"list":[{"current":{"selected":true,"text":"dev-cortex","value":"dev-cortex"},"description":null,"error":null,"hide":0,"includeAll":false,"label":null,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"allValue":".*","current":{"selected":false,"text":["default/grafana"],"value":["default/grafana"]},"datasource":"$datasource","definition":"label_values(grafana_build_info, job)","description":null,"error":null,"hide":0,"includeAll":true,"label":null,"multi":true,"name":"job","options":[],"query":{"query":"label_values(grafana_build_info, job)","refId":"Billing Admin-job-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"allValue":".*","current":{"selected":false,"text":"All","value":"$__all"},"datasource":"$datasource","definition":"label_values(grafana_build_info, instance)","description":null,"error":null,"hide":0,"includeAll":true,"label":null,"multi":true,"name":"instance","options":[],"query":{"query":"label_values(grafana_build_info, instance)","refId":"Billing Admin-instance-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-6h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone": "utc","title":"Grafana Overview","uid":"6be0s85Mk","version":2}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-coredns.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-k8s-coredns
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  k8s-coredns.json: |-
    {"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"description":"A dashboard for the CoreDNS DNS server with updated metrics for version 1.7.0+.  Based on the CoreDNS dashboard by buhay.","editable":true,"gnetId":12539,"graphTooltip":0,"iteration":1603798405693,"links":[{"icon":"external link","tags":[],"targetBlank":true,"title":"CoreDNS.io","type":"link","url":"https://coredns.io"}],"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":8,"x":0,"y":0},"hiddenSeries":false,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"total","yaxis":2}],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_request_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (proto) or\nsum(rate(coredns_dns_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (proto)","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}","refId":"A","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (total)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":8,"x":8,"y":0},"hiddenSeries":false,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"total","yaxis":2},{"alias":"other","yaxis":2}],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_request_type_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (type) or \nsum(rate(coredns_dns_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (type)","interval":"","intervalFactor":2,"legendFormat":"{{ type }}","refId":"A","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (by qtype)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":8,"x":16,"y":0},"hiddenSeries":false,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"total","yaxis":2}],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_request_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (zone) or\nsum(rate(coredns_dns_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (zone)","interval":"","intervalFactor":2,"legendFormat":"{{ zone }}","refId":"A","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (by zone)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":0,"y":7},"hiddenSeries":false,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"total","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_request_do_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_do_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m]))","interval":"","intervalFactor":2,"legendFormat":"DO","refId":"A","step":40},{"expr":"sum(rate(coredns_dns_request_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) or\nsum(rate(coredns_dns_requests_total{job=~\"$job\",instance=~\"$instance\"}[5m]))","interval":"","intervalFactor":2,"legendFormat":"total","refId":"B","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (DO bit)","tooltip":{"shared":true,"sort":2,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":6,"x":12,"y":7},"hiddenSeries":false,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"tcp:90","yaxis":2},{"alias":"tcp:99 ","yaxis":2},{"alias":"tcp:50","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:99 ","refId":"A","step":60},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))","intervalFactor":2,"legendFormat":"{{ proto }}:90","refId":"B","step":60},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))","intervalFactor":2,"legendFormat":"{{ proto }}:50","refId":"C","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (size, udp)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":6,"x":18,"y":7},"hiddenSeries":false,"id":12,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"tcp:90","yaxis":1},{"alias":"tcp:99 ","yaxis":1},{"alias":"tcp:50","yaxis":1}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:99 ","refId":"A","step":60},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:90","refId":"B","step":60},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:50","refId":"C","step":60}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Requests (size,tcp)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":0,"y":14},"hiddenSeries":false,"id":14,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_dns_response_rcode_count_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (rcode) or\nsum(rate(coredns_dns_responses_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (rcode)","interval":"","intervalFactor":2,"legendFormat":"{{ rcode }}","refId":"A","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Responses (by rcode)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":12,"y":14},"hiddenSeries":false,"id":32,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[5m])) by (le, job))","format":"time_series","intervalFactor":2,"legendFormat":"99%","refId":"A","step":40},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[5m])) by (le))","format":"time_series","intervalFactor":2,"legendFormat":"90%","refId":"B","step":40},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\"}[5m])) by (le))","format":"time_series","intervalFactor":2,"legendFormat":"50%","refId":"C","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Responses (duration)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":0,"y":21},"hiddenSeries":false,"id":18,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"udp:50%","yaxis":1},{"alias":"tcp:50%","yaxis":2},{"alias":"tcp:90%","yaxis":2},{"alias":"tcp:99%","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:99%","refId":"A","step":40},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ","interval":"","intervalFactor":2,"legendFormat":"{{ proto }}:90%","refId":"B","step":40},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ","hide":false,"intervalFactor":2,"legendFormat":"{{ proto }}:50%","metric":"","refId":"C","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Responses (size, udp)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":12,"y":21},"hiddenSeries":false,"id":20,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"udp:50%","yaxis":1},{"alias":"tcp:50%","yaxis":1},{"alias":"tcp:90%","yaxis":1},{"alias":"tcp:99%","yaxis":1}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ","format":"time_series","intervalFactor":2,"legendFormat":"{{ proto }}:99%","refId":"A","step":40},{"expr":"histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ","format":"time_series","intervalFactor":2,"legendFormat":"{{ proto }}:90%","refId":"B","step":40},{"expr":"histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{job=~\"$job\",instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ","format":"time_series","intervalFactor":2,"legendFormat":"{{ proto }}:50%","metric":"","refId":"C","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Responses (size, tcp)","tooltip":{"shared":true,"sort":0,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":0,"y":28},"hiddenSeries":false,"id":22,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(coredns_cache_size{job=~\"$job\",instance=~\"$instance\"}) by (type) or\nsum(coredns_cache_entries{job=~\"$job\",instance=~\"$instance\"}) by (type)","interval":"","intervalFactor":2,"legendFormat":"{{ type }}","refId":"A","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Cache (size)","tooltip":{"shared":true,"sort":2,"value_type":"cumulative"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"decbytes","logBase":1,"max":null,"min":0,"show":true},{"format":"short","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{},"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"grid":{},"gridPos":{"h":7,"w":12,"x":12,"y":28},"hiddenSeries":false,"id":24,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"percentage":false,"pluginVersion":"7.2.0","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"misses","yaxis":2}],"spaceLength":10,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(rate(coredns_cache_hits_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (type)","hide":false,"intervalFactor":2,"legendFormat":"hits:{{ type }}","refId":"A","step":40},{"expr":"sum(rate(coredns_cache_misses_total{job=~\"$job\",instance=~\"$instance\"}[5m])) by (type)","hide":false,"intervalFactor":2,"legendFormat":"misses","refId":"B","step":40}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Cache (hitrate)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","logBase":1,"max":null,"min":0,"show":true},{"format":"pps","logBase":1,"max":null,"min":0,"show":true}],"yaxis":{"align":false,"alignLevel":null}}],"refresh":"10s","schemaVersion":26,"style":"dark","tags":["dns","coredns"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"includeAll":false,"label":null,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"allValue":".*","current":{"selected":false,"text":"coredns","value":"coredns"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(coredns_dns_requests_total, job)","hide":0,"includeAll":true,"label":"Job","multi":false,"name":"job","options":[],"query":{"query":"label_values(coredns_dns_requests_total, job)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"allValue":".*","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","definition":"label_values(coredns_dns_requests_total{job=~\"$job\"}, instance)","hide":0,"includeAll":true,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(coredns_dns_requests_total{job=~\"$job\"}, instance)","refresh":1,"regex":"","skipUrlSync":false,"sort":3,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-3h","to":"now"},"timepicker":{"refresh_intervals":["10s","30s","1m","5m","15m","30m","1h","2h","1d"]},"timezone":"utc","title":"CoreDNS","uid":"vkQ0UHxik","version":3}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-cluster.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-k8s-resources-cluster
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  k8s-resources-cluster.json: |-
    {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m{cluster=\"$cluster\"}","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"cpu\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\",resource=\"memory\",cluster=\"$cluster\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Pods","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__data.fields.namespace}","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"Workloads","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to workloads","linkUrl":"/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__data.fields.namespace}","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"G"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Pods","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__data.fields.namespace}","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"Workloads","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to workloads","linkUrl":"/d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__data.fields.namespace}","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(kube_pod_owner{job=\"kube-state-metrics\", cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"count(avg(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{cluster=\"$cluster\"}) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"G"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Namespace","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Current Receive Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Transmit Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Network Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Network Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":14,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"avg(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Namespace: Received","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":15,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"avg(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Namespace: Transmitted","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Container Bandwidth by Namespace","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":16,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":17,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":18,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":19,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=~\".+\"}[$__rate_interval])) by (namespace)","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":null,"fill":10,"id":20,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"ceil(sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval])))","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"IOPS(Reads+Writes)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":21,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{namespace}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ThroughPut(Read+Write)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":22,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"sort":{"col":4,"desc":true},"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"IOPS(Reads)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Reads + Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"iops"},{"alias":"Throughput(Read)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Read + Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Namespace","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=${__value.text}","pattern":"namespace","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum by(namespace) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum by(namespace) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum by(namespace) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum by(namespace) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace!=\"\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Storage IO","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Cluster","uid":"efa86fd1d0c121a26444b636a3f509a8","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-multicluster.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-k8s-resources-multicluster
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  k8s-resources-multicluster.json: |-
    {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"cluster:node_cpu:ratio_rate5m","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"1 - sum(:node_memory_MemAvailable_bytes:sum) / sum(node_memory_MemTotal_bytes{job=\"node-exporter\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Requests Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":2,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) / sum(kube_node_status_allocatable{job=\"kube-state-metrics\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Limits Commitment","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=${__value.text}","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"cpu\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"time_series","legendFormat":"{{cluster}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/efa86fd1d0c121a26444b636a3f509a8/k8s-resources-cluster?var-datasource=$datasource&var-cluster=${__value.text}","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\"}) by (cluster) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", resource=\"memory\"}) by (cluster)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Requests by Cluster","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Requests","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources /  Multi-Cluster","uid":"b59e6c9f2fcbe2e16d77fc492374cc4f","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-namespace.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-k8s-resources-namespace
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  k8s-resources-namespace.json: |-
    {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"100px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation (from requests)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"CPU Utilisation (from limits)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation (from requests)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"format":"percentunit","id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":3,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) / sum(kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"})","format":"time_series","instant":true,"refId":"A"}],"thresholds":"70,80","timeFrom":null,"timeShift":null,"title":"Memory Utilisation (from limits)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"singlestat","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Headlines","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"quota - requests","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"quota - limits","color":"#FF9830","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})","format":"time_series","legendFormat":"quota - requests","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})","format":"time_series","legendFormat":"quota - limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"quota - requests","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"quota - limits","color":"#FF9830","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})","format":"time_series","legendFormat":"quota - requests","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})","format":"time_series","legendFormat":"quota - limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Usage (RSS)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Cache)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Swap)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\", image!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(container_memory_cache{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"G"},{"expr":"sum(container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"H"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Current Receive Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Transmit Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Network Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Network Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":14,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":15,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":null,"fill":10,"id":16,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"ceil(sum by(pod) (rate(container_fs_reads_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"IOPS(Reads+Writes)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":17,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by(pod) (rate(container_fs_reads_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{container!=\"\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ThroughPut(Read+Write)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":18,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"sort":{"col":4,"desc":true},"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"IOPS(Reads)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Reads + Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"iops"},{"alias":"Throughput(Read)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Read + Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Storage IO","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Namespace (Pods)","uid":"85a562078cdf77779eaa1add43ccec1e","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-node.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-k8s-resources-node
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  k8s-resources-node.json: |-
    {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"max capacity","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"cpu\"})","format":"time_series","legendFormat":"max capacity","legendLink":null},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", node=~\"$node\"}) by (pod) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"max capacity","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(kube_node_status_capacity{cluster=\"$cluster\", node=~\"$node\", resource=\"memory\"})","format":"time_series","legendFormat":"max capacity","legendLink":null},{"expr":"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\", container!=\"\"}) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (w/o cache)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Usage (RSS)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Cache)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Swap)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", node=~\"$node\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"G"},{"expr":"sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=~\"$node\",container!=\"\"}) by (pod)","format":"table","instant":true,"legendFormat":"","refId":"H"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":true,"name":"node","options":[],"query":"label_values(kube_node_info{cluster=\"$cluster\"}, node)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Node (Pods)","uid":"200ac8fdbfbb74b39aff88118e4d1c2c","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-pod.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-k8s-resources-pod
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  k8s-resources-pod.json: |-
    {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"requests","color":"#F2495C","fill":0,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"limits","color":"#FF9830","fill":0,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace=\"$namespace\", pod=\"$pod\", cluster=\"$cluster\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null},{"expr":"sum(\n    kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n","format":"time_series","legendFormat":"requests","legendLink":null},{"expr":"sum(\n    kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"cpu\"}\n)\n","format":"time_series","legendFormat":"limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":true,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(increase(container_cpu_cfs_throttled_periods_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container) /sum(increase(container_cpu_cfs_periods_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", cluster=\"$cluster\"}[$__rate_interval])) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[{"colorMode":"critical","fill":true,"line":true,"op":"gt","value":0.25,"yaxis":"left"}],"timeFrom":null,"timeShift":null,"title":"CPU Throttling","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Throttling","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"requests","color":"#F2495C","dashes":true,"fill":0,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"limits","color":"#FF9830","dashes":true,"fill":0,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)","format":"time_series","legendFormat":"{{container}}","legendLink":null},{"expr":"sum(\n    kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n","format":"time_series","legendFormat":"requests","legendLink":null},{"expr":"sum(\n    kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", resource=\"memory\"}\n)\n","format":"time_series","legendFormat":"limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage (WSS)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage (WSS)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Usage (RSS)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Cache)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Usage (Swap)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\", image!=\"\"}) by (container) / sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(container_memory_rss{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"F"},{"expr":"sum(container_memory_cache{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"G"},{"expr":"sum(container_memory_swap{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)","format":"table","instant":true,"legendFormat":"","refId":"H"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])) by (pod)","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":null,"fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"ceil(sum by(pod) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))","format":"time_series","legendFormat":"Reads","legendLink":null},{"expr":"ceil(sum by(pod) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\",namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval])))","format":"time_series","legendFormat":"Writes","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"IOPS","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by(pod) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))","format":"time_series","legendFormat":"Reads","legendLink":null},{"expr":"sum by(pod) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))","format":"time_series","legendFormat":"Writes","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ThroughPut","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution(Pod - Read & Writes)","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","decimals":null,"fill":10,"id":14,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"ceil(sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval])))","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"IOPS(Reads+Writes)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":15,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"time_series","legendFormat":"{{container}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"ThroughPut(Read+Write)","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution(Containers)","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":16,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"sort":{"col":4,"desc":true},"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"IOPS(Reads)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"iops"},{"alias":"IOPS(Reads + Writes)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":3,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"iops"},{"alias":"Throughput(Read)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Throughput(Read + Write)","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Container","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"container","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum by(container) (rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\",device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum by(container) (rate(container_fs_reads_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum by(container) (rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum by(container) (rate(container_fs_reads_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]) + rate(container_fs_writes_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\", container!=\"\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Storage IO","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage IO - Distribution","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"pod","options":[],"query":"label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Pod","uid":"6581e46e4e5c7ba40a07646395ef7b23","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-workload.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-k8s-resources-workload
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  k8s-resources-workload.json: |-
    {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(\n    node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(\n    node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(\n    kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(\n    node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n/sum(\n    kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(\n    kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(\n    node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n/sum(\n    kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(\n    container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"sum(\n    container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(\n    kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(\n    container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n/sum(\n    kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(\n    kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(\n    container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n/sum(\n    kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=~\"$type\"}\n) by (pod)\n","format":"table","instant":true,"legendFormat":"","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Current Receive Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Transmit Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Network Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Network Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(avg(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Pod: Received","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(avg(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Pod: Transmitted","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Container Bandwidth by Pod","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","legendFormat":"{{pod}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":true,"label":null,"multi":false,"name":"type","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\"}, workload_type)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"workload","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}, workload)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Workload","uid":"a164a7f0339f99e89cea5cb47e9be617","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/k8s-resources-workloads-namespace.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-k8s-resources-workloads-namespace
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  k8s-resources-workloads-namespace.json: |-
    {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"10s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":1,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"quota - requests","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"quota - limits","color":"#FF9830","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(\n  node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"time_series","legendFormat":"{{workload}} - {{workload_type}}","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.cpu\"})","format":"time_series","legendFormat":"quota - requests","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.cpu\"})","format":"time_series","legendFormat":"quota - limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Running Pods","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"CPU Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"short"},{"alias":"CPU Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Workload","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=${__value.text}&var-type=${__data.fields.workload_type}","pattern":"workload","thresholds":[],"type":"number","unit":"short"},{"alias":"Workload Type","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"workload_type","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload, workload_type)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(\n  node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(\n  kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(\n  node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n/sum(\n  kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(\n  kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(\n  node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n/sum(\n  kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"quota - requests","color":"#F2495C","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false},{"alias":"quota - limits","color":"#FF9830","dashes":true,"fill":0,"hiddenSeries":true,"hideTooltip":true,"legend":true,"linewidth":2,"stack":false}],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(\n    container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"time_series","legendFormat":"{{workload}} - {{workload_type}}","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"requests.memory\"})","format":"time_series","legendFormat":"quota - requests","legendLink":null},{"expr":"scalar(kube_resourcequota{cluster=\"$cluster\", namespace=\"$namespace\", type=\"hard\",resource=\"limits.memory\"})","format":"time_series","legendFormat":"quota - limits","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Running Pods","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"short"},{"alias":"Memory Usage","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Requests %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Memory Limits","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"bytes"},{"alias":"Memory Limits %","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"percentunit"},{"alias":"Workload","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=${__value.text}&var-type=${__data.fields.workload_type}","pattern":"workload","thresholds":[],"type":"number","unit":"short"},{"alias":"Workload Type","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"workload_type","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"count(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload, workload_type)","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"sum(\n    container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"sum(\n  kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"sum(\n    container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n/sum(\n  kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"sum(\n  kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"sum(\n    container_memory_working_set_bytes{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\", image!=\"\"}\n  * on(namespace,pod)\n    group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n/sum(\n  kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\"}\n* on(namespace,pod)\n  group_left(workload, workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}\n) by (workload, workload_type)\n","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Quota","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory Quota","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Current Receive Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Transmit Bandwidth","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Workload","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTargetBlank":false,"linkTooltip":"Drill down to pods","linkUrl":"/d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=${__value.text}&var-type=$type","pattern":"workload","thresholds":[],"type":"number","unit":"short"},{"alias":"Workload Type","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"workload_type","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"B"},{"expr":"(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"C"},{"expr":"(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"D"},{"expr":"(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"E"},{"expr":"(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"legendFormat":"","refId":"F"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Network Usage","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Network Usage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(avg(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Workload: Received","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(avg(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Container Bandwidth by Workload: Transmitted","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Container Bandwidth by Workload","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":11,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":12,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":13,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","legendFormat":"{{workload}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":false,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Rate of Packets Dropped","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"type","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\", namespace=\"$namespace\", workload=~\".+\"}, workload_type)","refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Compute Resources / Namespace (Workloads)","uid":"a87fb0d919ec0ea5f6543124e16c42a5","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/kubelet.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-kubelet
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  kubelet.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":2,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(kubelet_node_name{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"})","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"title":"Running Kubelets","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":4,"y":0},"id":3,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(kubelet_running_pods{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Running Pods","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":8,"y":0},"id":4,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(kubelet_running_containers{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}) OR sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Running Containers","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":12,"y":0},"id":5,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\", state=\"actual_state_of_world\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Actual Volume Count","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":16,"y":0},"id":6,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",state=\"desired_state_of_world\"})","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Desired Volume Count","transparent":false,"type":"stat"},{"datasource":"$datasource","fieldConfig":{"defaults":{"links":[],"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"none"}},"gridPos":{"h":7,"w":4,"x":20,"y":0},"id":7,"links":[],"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"7","targets":[{"expr":"sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"title":"Config Error Count","transparent":false,"type":"stat"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":7},"id":8,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (operation_type, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Operation Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":7},"id":9,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Operation Error Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":14},"id":10,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Operation duration 99th quantile","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":21},"id":11,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} pod","refId":"A"},{"expr":"sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} worker","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Pod Start Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":21},"id":12,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} pod","refId":"A"},{"expr":"histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} worker","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Pod Start Duration","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":28},"id":13,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Storage Operation Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":28},"id":14,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Storage Operation Error Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":35},"id":15,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_name, volume_plugin, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_name}} {{volume_plugin}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Storage Operation Duration 99th quantile","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":42},"id":16,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type)","format":"time_series","intervalFactor":2,"legendFormat":"{{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Cgroup manager operation rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":42},"id":17,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, operation_type, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{operation_type}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Cgroup manager 99th quantile","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Pod lifecycle event generator","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":0,"y":49},"id":18,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"PLEG relist rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":12,"x":12,"y":49},"id":19,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"PLEG relist interval","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":56},"id":20,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])) by (instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"PLEG relist duration","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":63},"id":21,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"2xx","refId":"A"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"3xx","refId":"B"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"4xx","refId":"C"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"5xx","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"RPC Rate","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":70},"id":22,"legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\", instance=~\"$instance\"}[$__rate_interval])) by (instance, verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Request duration 99th quantile","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":8,"x":0,"y":77},"id":23,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":8,"x":8,"y":77},"id":24,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}[$__rate_interval])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{"h":7,"w":8,"x":16,"y":77},"id":25,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"go_goroutines{cluster=\"$cluster\",job=\"kubelet\", metrics_path=\"/metrics\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Goroutines","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"refresh":"10s","rows":[],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":true,"label":"instance","multi":false,"name":"instance","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics\",cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Kubelet","uid":"3138fa155d5915769fbded898ac09fd9","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/namespace-by-pod.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-namespace-by-pod
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  namespace-by-pod.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":0,"format":"time_series","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":9,"w":12,"x":0,"y":1},"height":9,"id":3,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"minSpan":12,"nullPointMode":"connected","nullText":null,"options":{"fieldOptions":{"calcs":["last"],"defaults":{"max":10000000000,"min":0,"title":"$namespace","unit":"Bps"},"mappings":[],"override":{},"thresholds":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}],"values":false}},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":12,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))","format":"time_series","instant":null,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","type":"gauge","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":0,"format":"time_series","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":9,"w":12,"x":12,"y":1},"height":9,"id":4,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"minSpan":12,"nullPointMode":"connected","nullText":null,"options":{"fieldOptions":{"calcs":["last"],"defaults":{"max":10000000000,"min":0,"title":"$namespace","unit":"Bps"},"mappings":[],"override":{},"thresholds":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}],"values":false}},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":12,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution]))","format":"time_series","instant":null,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","type":"gauge","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"columns":[{"text":"Time","value":"Time"},{"text":"Value #A","value":"Value #A"},{"text":"Value #B","value":"Value #B"},{"text":"Value #C","value":"Value #C"},{"text":"Value #D","value":"Value #D"},{"text":"Value #E","value":"Value #E"},{"text":"Value #F","value":"Value #F"},{"text":"pod","value":"pod"}],"datasource":"$datasource","fill":1,"fontSize":"100%","gridPos":{"h":9,"w":24,"x":0,"y":10},"id":5,"lines":true,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null as zero","renderer":"flot","scroll":true,"showHeader":true,"sort":{"col":0,"desc":false},"spaceLength":10,"span":24,"styles":[{"alias":"Time","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Time","thresholds":[],"type":"hidden","unit":"short"},{"alias":"Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Pod","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTooltip":"Drill down","linkUrl":"d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=${__value.text}","pattern":"pod","thresholds":[],"type":"number","unit":"short"}],"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"A","step":10},{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"B","step":10},{"expr":"sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"C","step":10},{"expr":"sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"D","step":10},{"expr":"sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"E","step":10},{"expr":"sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"F","step":10}],"timeFrom":null,"timeShift":null,"title":"Current Status","type":"table"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":19},"id":6,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":20},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":20},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":9,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":0,"y":30},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":12,"y":30},"id":11,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":12,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":0,"y":40},"id":13,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":12,"y":40},"id":14,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"kube-system","value":"kube-system"},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Namespace (Pods)","uid":"8b7a8b326d7a6f1f04244066368c67af","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/namespace-by-workload.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-namespace-by-workload
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  namespace-by-workload.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":1},"id":3,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ workload }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":1},"id":4,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ workload }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"columns":[{"text":"Time","value":"Time"},{"text":"Value #A","value":"Value #A"},{"text":"Value #B","value":"Value #B"},{"text":"Value #C","value":"Value #C"},{"text":"Value #D","value":"Value #D"},{"text":"Value #E","value":"Value #E"},{"text":"Value #F","value":"Value #F"},{"text":"Value #G","value":"Value #G"},{"text":"Value #H","value":"Value #H"},{"text":"workload","value":"workload"}],"datasource":"$datasource","fill":1,"fontSize":"90%","gridPos":{"h":9,"w":24,"x":0,"y":10},"id":5,"lines":true,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null as zero","renderer":"flot","scroll":true,"showHeader":true,"sort":{"col":0,"desc":false},"spaceLength":10,"span":24,"styles":[{"alias":"Time","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Time","thresholds":[],"type":"hidden","unit":"short"},{"alias":"Current Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Current Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Average Bandwidth Received","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #C","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Average Bandwidth Transmitted","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #D","thresholds":[],"type":"number","unit":"Bps"},{"alias":"Rate of Received Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #E","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #F","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Received Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #G","thresholds":[],"type":"number","unit":"pps"},{"alias":"Rate of Transmitted Packets Dropped","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #H","thresholds":[],"type":"number","unit":"pps"},{"alias":"Workload","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":true,"linkTooltip":"Drill down","linkUrl":"d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=${__value.text}","pattern":"workload","thresholds":[],"type":"number","unit":"short"}],"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"A","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"B","step":10},{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"C","step":10},{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"D","step":10},{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"E","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"F","step":10},{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"G","step":10},{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":"","refId":"H","step":10}],"timeFrom":null,"timeShift":null,"title":"Current Status","type":"table"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":19},"id":6,"panels":[{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":20},"id":7,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ workload }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":20},"id":8,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ workload }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Bandwidth","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":29},"id":9,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth HIstory","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":38},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":38},"id":11,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":39},"id":12,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":40},"id":13,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":40},"id":14,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":40},"id":15,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":41},"id":16,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":41},"id":17,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\", workload_type=~\"$type\"}) by (workload))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{workload}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"kube-system","value":"kube-system"},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","hide":0,"includeAll":false,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"type","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=\"$namespace\", workload=~\".+\"}, workload_type)","refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Namespace (Workload)","uid":"bbb2a765a623ae38130206c7d94a160f","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/node-cluster-rsrc-use.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-node-cluster-rsrc-use
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  node-cluster-rsrc-use.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"((\n  instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n  *\n  instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}\n) != 0 )\n/ scalar(sum(instance:node_num_cpu:sum{job=\"node-exporter\", cluster=\"$cluster\"}))\n","format":"time_series","intervalFactor":2,"legendFormat":"{{ instance }}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n  / scalar(count(instance:node_load1_per_cpu:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n)  != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Saturation (Load1 per CPU)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}\n  / scalar(count(instance:node_memory_utilisation:ratio{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Major Page Faults)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/Receive/","stack":"A"},{"alias":"/Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Receive","refId":"A"},{"expr":"instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Utilisation (Bytes Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ Receive/","stack":"A"},{"alias":"/ Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Receive","refId":"A"},{"expr":"instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Saturation (Drops Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n  / scalar(count(instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}\n  / scalar(count(instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", cluster=\"$cluster\"}))\n) != 0\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}} {{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Saturation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk IO","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum without (device) (\n  max without (fstype, mountpoint) ((\n    node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n    -\n    node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"}\n  ) != 0)\n)\n/ scalar(sum(max without (fstype, mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", mountpoint!=\"\", cluster=\"$cluster\"})))\n","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Space Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk Space","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(node_time_seconds, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Node Exporter / USE Method / Cluster","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/node-rsrc-use.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-node-rsrc-use
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  node-rsrc-use.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_cpu_utilisation:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Utilisation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Saturation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Saturation (Load1 per CPU)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_memory_utilisation:ratio{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Utilisation","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":5,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_vmstat_pgmajfault:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Major page Faults","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Saturation (Major Page Faults)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"rds","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/Receive/","stack":"A"},{"alias":"/Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Receive","refId":"A"},{"expr":"instance:node_network_transmit_bytes_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Utilisation (Bytes Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ Receive/","stack":"A"},{"alias":"/ Transmit/","stack":"B","transform":"negative-Y"}],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance:node_network_receive_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Receive","refId":"A"},{"expr":"instance:node_network_transmit_drop_excluding_lo:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"Transmit","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Saturation (Drops Receive/Transmit)","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance_device:node_disk_io_time_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"instance_device:node_disk_io_time_weighted_seconds:rate5m{job=\"node-exporter\", instance=\"$instance\", cluster=\"$cluster\"} != 0","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk IO Saturation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk IO","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"fillGradient":0,"gridPos":{},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":false,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(1 -\n  (\n   max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n   /\n   max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\", cluster=\"$cluster\"})\n  ) != 0\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk Space Utilisation","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk Space","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{"text":"","value":""},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(node_time_seconds, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(node_exporter_build_info{job=\"node-exporter\", cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Node Exporter / USE Method / Node","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/nodes-darwin.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-nodes-darwin
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  nodes-darwin.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n  count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n","format":"time_series","intervalFactor":5,"legendFormat":"{{cpu}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node_load1{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"1m load average","refId":"A"},{"expr":"node_load5{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"5m load average","refId":"B"},{"expr":"node_load15{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"15m load average","refId":"C"},{"expr":"count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})","format":"time_series","intervalFactor":2,"legendFormat":"logical cores","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Load Average","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":false,"steppedLine":false,"targets":[{"expr":"node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"Physical Memory","refId":"A"},{"expr":"(\n    node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n    node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n    node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"} +\n    node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"Memory Used","refId":"B"},{"expr":"(\n    node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"} -\n    node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"App Memory","refId":"C"},{"expr":"node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"Wired Memory","refId":"D"},{"expr":"node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"Compressed","refId":"E"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"}},"gridPos":{},"id":5,"span":3,"targets":[{"expr":"(\n    (\n      avg(node_memory_internal_bytes{job=\"node-exporter\", instance=\"$instance\"}) -\n      avg(node_memory_purgeable_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n      avg(node_memory_wired_bytes{job=\"node-exporter\", instance=\"$instance\"}) +\n      avg(node_memory_compressed_bytes{job=\"node-exporter\", instance=\"$instance\"})\n    ) /\n    avg(node_memory_total_bytes{job=\"node-exporter\", instance=\"$instance\"})\n)\n*\n100\n","format":"time_series","intervalFactor":2,"legendFormat":""}],"title":"Memory Usage","transparent":false,"type":"gauge"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ read| written/","yaxis":1},{"alias":"/ io time/","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} read","refId":"A"},{"expr":"rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} written","refId":"B"},{"expr":"rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{},"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"yellow","value":0.8},{"color":"red","value":0.9}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byName","options":"Mounted on"},"properties":[{"id":"custom.width","value":260}]},{"matcher":{"id":"byName","options":"Size"},"properties":[{"id":"custom.width","value":93}]},{"matcher":{"id":"byName","options":"Used"},"properties":[{"id":"custom.width","value":72}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"custom.width","value":88}]},{"matcher":{"id":"byName","options":"Used, %"},"properties":[{"id":"unit","value":"percentunit"},{"id":"custom.displayMode","value":"gradient-gauge"},{"id":"max","value":1},{"id":"min","value":0}]}]},"gridPos":{},"id":7,"span":6,"targets":[{"expr":"max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""},{"expr":"max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""}],"title":"Disk Space Usage","transformations":[{"id":"groupBy","options":{"fields":{"Value #A":{"aggregations":["lastNotNull"],"operation":"aggregate"},"Value #B":{"aggregations":["lastNotNull"],"operation":"aggregate"},"mountpoint":{"aggregations":[],"operation":"groupby"}}}},{"id":"merge","options":{}},{"id":"calculateField","options":{"alias":"Used","binary":{"left":"Value #A (lastNotNull)","operator":"-","reducer":"sum","right":"Value #B (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"calculateField","options":{"alias":"Used, %","binary":{"left":"Used","operator":"/","reducer":"sum","right":"Value #A (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Value #A (lastNotNull)":"Size","Value #B (lastNotNull)":"Available","mountpoint":"Mounted on"}}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"Mounted on"}]}}],"transparent":false,"type":"table"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network received (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Received","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network transmitted (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Transmitted","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(node_uname_info{job=\"node-exporter\", sysname=\"Darwin\"}, instance)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Node Exporter / MacOS","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/nodes.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-nodes
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  nodes.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":1,"hideControls":false,"id":null,"links":[],"refresh":"30s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":2,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  (1 - sum without (mode) (rate(node_cpu_seconds_total{job=\"node-exporter\", mode=~\"idle|iowait|steal\", instance=\"$instance\"}[$__rate_interval])))\n/ ignoring(cpu) group_left\n  count without (cpu, mode) (node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n","format":"time_series","intervalFactor":5,"legendFormat":"{{cpu}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":1,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":3,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"node_load1{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"1m load average","refId":"A"},{"expr":"node_load5{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"5m load average","refId":"B"},{"expr":"node_load15{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"15m load average","refId":"C"},{"expr":"count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})","format":"time_series","intervalFactor":2,"legendFormat":"logical cores","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Load Average","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"CPU","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n  node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n  node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n  node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n","format":"time_series","intervalFactor":2,"legendFormat":"memory used","refId":"A"},{"expr":"node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory buffers","refId":"B"},{"expr":"node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory cached","refId":"C"},{"expr":"node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"memory free","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory Usage","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"rgba(50, 172, 45, 0.97)"},{"color":"rgba(237, 129, 40, 0.89)","value":80},{"color":"rgba(245, 54, 54, 0.9)","value":90}]},"unit":"percent"}},"gridPos":{},"id":5,"span":3,"targets":[{"expr":"100 -\n(\n  avg(node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}) /\n  avg(node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"})\n* 100\n)\n","format":"time_series","intervalFactor":2,"legendFormat":""}],"title":"Memory Usage","transparent":false,"type":"gauge"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Memory","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":0,"fillGradient":0,"gridPos":{},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[{"alias":"/ read| written/","yaxis":1},{"alias":"/ io time/","yaxis":2}],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} read","refId":"A"},{"expr":"rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} written","refId":"B"},{"expr":"rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)\"}[$__rate_interval])","format":"time_series","intervalFactor":1,"legendFormat":"{{device}} io time","refId":"C"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Disk I/O","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"percentunit","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"datasource":"$datasource","fieldConfig":{"defaults":{"custom":{},"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"yellow","value":0.8},{"color":"red","value":0.9}]},"unit":"decbytes"},"overrides":[{"matcher":{"id":"byName","options":"Mounted on"},"properties":[{"id":"custom.width","value":260}]},{"matcher":{"id":"byName","options":"Size"},"properties":[{"id":"custom.width","value":93}]},{"matcher":{"id":"byName","options":"Used"},"properties":[{"id":"custom.width","value":72}]},{"matcher":{"id":"byName","options":"Available"},"properties":[{"id":"custom.width","value":88}]},{"matcher":{"id":"byName","options":"Used, %"},"properties":[{"id":"unit","value":"percentunit"},{"id":"custom.displayMode","value":"gradient-gauge"},{"id":"max","value":1},{"id":"min","value":0}]}]},"gridPos":{},"id":7,"span":6,"targets":[{"expr":"max by (mountpoint) (node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""},{"expr":"max by (mountpoint) (node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\", mountpoint!=\"\"})\n","format":"table","instant":true,"intervalFactor":2,"legendFormat":""}],"title":"Disk Space Usage","transformations":[{"id":"groupBy","options":{"fields":{"Value #A":{"aggregations":["lastNotNull"],"operation":"aggregate"},"Value #B":{"aggregations":["lastNotNull"],"operation":"aggregate"},"mountpoint":{"aggregations":[],"operation":"groupby"}}}},{"id":"merge","options":{}},{"id":"calculateField","options":{"alias":"Used","binary":{"left":"Value #A (lastNotNull)","operator":"-","reducer":"sum","right":"Value #B (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"calculateField","options":{"alias":"Used, %","binary":{"left":"Used","operator":"/","reducer":"sum","right":"Value #A (lastNotNull)"},"mode":"binary","reduce":{"reducer":"sum"}}},{"id":"organize","options":{"excludeByName":{},"indexByName":{},"renameByName":{"Value #A (lastNotNull)":"Size","Value #B (lastNotNull)":"Available","mountpoint":"Mounted on"}}},{"id":"sortBy","options":{"fields":{},"sort":[{"field":"Mounted on"}]}}],"transparent":false,"type":"table"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Disk","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network received (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":8,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Received","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","description":"Network transmitted (bits/s)","fill":0,"fillGradient":0,"gridPos":{},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__rate_interval]) * 8","format":"time_series","intervalFactor":1,"legendFormat":"{{device}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Network Transmitted","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Network","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["node-exporter-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Instance","multi":false,"name":"instance","options":[],"query":"label_values(node_uname_info{job=\"node-exporter\", sysname!=\"Darwin\"}, instance)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Node Exporter / Nodes","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/persistentvolumesusage.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-persistentvolumesusage
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  persistentvolumesusage.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"10s","rows":[{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":2,"interval":"1m","legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":true,"steppedLine":false,"targets":[{"expr":"(\n  sum without(instance, node) (topk(1, (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n  -\n  sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n","format":"time_series","intervalFactor":1,"legendFormat":"Used Space","refId":"A"},{"expr":"sum without(instance, node) (topk(1, (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n","format":"time_series","intervalFactor":1,"legendFormat":"Free Space","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Volume Space Usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"$datasource","format":"percent","gauge":{"maxValue":100,"minValue":0,"show":true,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":3,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"max without(instance,node) (\n(\n  topk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n  -\n  topk(1, kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n/\ntopk(1, kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"80, 90","title":"Volume Space Usage","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":true,"current":true,"max":true,"min":true,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":9,"stack":true,"steppedLine":false,"targets":[{"expr":"sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n","format":"time_series","intervalFactor":1,"legendFormat":"Used inodes","refId":"A"},{"expr":"(\n  sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n  -\n  sum without(instance, node) (topk(1, (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})))\n)\n","format":"time_series","intervalFactor":1,"legendFormat":" Free inodes","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Volume inodes Usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"none","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"none","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["rgba(50, 172, 45, 0.97)","rgba(237, 129, 40, 0.89)","rgba(245, 54, 54, 0.9)"],"datasource":"$datasource","format":"percent","gauge":{"maxValue":100,"minValue":0,"show":true,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":5,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":3,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"max without(instance,node) (\ntopk(1, kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n/\ntopk(1, kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n* 100)\n","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"80, 90","title":"Volume inodes Usage","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(kubelet_volume_stats_capacity_bytes{job=\"kubelet\", metrics_path=\"/metrics\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"Namespace","multi":false,"name":"namespace","options":[],"query":"label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\"}, namespace)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":false,"label":"PersistentVolumeClaim","multi":false,"name":"volume","options":[],"query":"label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=\"$namespace\"}, persistentvolumeclaim)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-7d","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Persistent Volumes","uid":"919b92a8e8041bd567af9edab12c840c","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/pod-total.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-pod-total
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  pod-total.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":0,"format":"time_series","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":9,"w":12,"x":0,"y":1},"height":9,"id":3,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"minSpan":12,"nullPointMode":"connected","nullText":null,"options":{"fieldOptions":{"calcs":["last"],"defaults":{"max":10000000000,"min":0,"title":"$namespace: $pod","unit":"Bps"},"mappings":[],"override":{},"thresholds":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}],"values":false}},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":12,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))","format":"time_series","instant":null,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","type":"gauge","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","decimals":0,"format":"time_series","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":9,"w":12,"x":12,"y":1},"height":9,"id":4,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"minSpan":12,"nullPointMode":"connected","nullText":null,"options":{"fieldOptions":{"calcs":["last"],"defaults":{"max":10000000000,"min":0,"title":"$namespace: $pod","unit":"Bps"},"mappings":[],"override":{},"thresholds":[{"color":"dark-green","index":0,"value":null},{"color":"dark-yellow","index":1,"value":5000000000},{"color":"dark-red","index":2,"value":7000000000}],"values":false}},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":12,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))","format":"time_series","instant":null,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","type":"gauge","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":10},"id":5,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":11},"id":6,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":11},"id":7,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":20},"id":8,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":0,"y":21},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":12,"y":21},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":11,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":0,"y":32},"id":12,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":10,"w":12,"x":12,"y":32},"id":13,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\",namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"kube-system","value":"kube-system"},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(container_network_receive_packets_total{cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)","hide":0,"includeAll":false,"label":null,"multi":false,"name":"pod","options":[],"query":"label_values(container_network_receive_packets_total{cluster=\"$cluster\",namespace=~\"$namespace\"}, pod)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Pod","uid":"7a18067ce943a40ae25454675c19ff5c","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/prometheus.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-prometheus
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  prometheus.json: |-
    {"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"links":[],"refresh":"60s","rows":[{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":1,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"styles":[{"alias":"Time","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Count","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #A","thresholds":[],"type":"hidden","unit":"short"},{"alias":"Uptime","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"Value #B","thresholds":[],"type":"number","unit":"s"},{"alias":"Cluster","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"cluster","thresholds":[],"type":"number","unit":"short"},{"alias":"Instance","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"instance","thresholds":[],"type":"number","unit":"short"},{"alias":"Job","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"job","thresholds":[],"type":"number","unit":"short"},{"alias":"Version","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"linkTargetBlank":false,"linkTooltip":"Drill down","linkUrl":"","pattern":"version","thresholds":[],"type":"number","unit":"short"},{"alias":"","colorMode":null,"colors":[],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"pattern":"/.*/","thresholds":[],"type":"string","unit":"short"}],"targets":[{"expr":"count by (cluster, job, instance, version) (prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})","format":"table","instant":true,"legendFormat":"","refId":"A"},{"expr":"max by (cluster, job, instance) (time() - process_start_time_seconds{cluster=~\"$cluster\", job=~\"$job\", instance=~\"$instance\"})","format":"table","instant":true,"legendFormat":"","refId":"B"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Prometheus Stats","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"transform":"table","type":"table","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Prometheus Stats","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":2,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(prometheus_target_sync_length_seconds_sum{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[5m])) by (cluster, job, scrape_job, instance) * 1e3","format":"time_series","legendFormat":"{{cluster}}:{{job}}:{{instance}}:{{scrape_job}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Target Sync","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ms","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":3,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (cluster, job, instance) (prometheus_sd_discovered_targets{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"})","format":"time_series","legendFormat":"{{cluster}}:{{job}}:{{instance}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Targets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Discovery","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"id":4,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(prometheus_target_interval_length_seconds_sum{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3","format":"time_series","legendFormat":"{{cluster}}:{{job}}:{{instance}} {{interval}} configured","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Scrape Interval Duration","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ms","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":5,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":true,"steppedLine":false,"targets":[{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_body_size_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"exceeded body size limit: {{cluster}} {{job}} {{instance}}","legendLink":null},{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"exceeded sample limit: {{cluster}} {{job}} {{instance}}","legendLink":null},{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"duplicate timestamp: {{cluster}} {{job}} {{instance}}","legendLink":null},{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"out of bounds: {{cluster}} {{job}} {{instance}}","legendLink":null},{"expr":"sum by (cluster, job, instance) (rate(prometheus_target_scrapes_sample_out_of_order_total{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}[1m]))","format":"time_series","legendFormat":"out of order: {{cluster}} {{job}} {{instance}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Scrape failures","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":4,"stack":true,"steppedLine":false,"targets":[{"expr":"rate(prometheus_tsdb_head_samples_appended_total{cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}[5m])","format":"time_series","legendFormat":"{{cluster}} {{job}} {{instance}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Appended Samples","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Retrieval","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"prometheus_tsdb_head_series{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}","format":"time_series","legendFormat":"{{cluster}} {{job}} {{instance}} head series","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Head Series","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":8,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"prometheus_tsdb_head_chunks{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\"}","format":"time_series","legendFormat":"{{cluster}} {{job}} {{instance}} head chunks","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Head Chunks","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Storage","titleSize":"h6"},{"collapse":false,"height":"250px","panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"rate(prometheus_engine_query_duration_seconds_count{cluster=~\"$cluster\",job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])","format":"time_series","legendFormat":"{{cluster}} {{job}} {{instance}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Query Rate","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":10,"id":10,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":0,"links":[],"nullPointMode":"null as zero","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"span":6,"stack":true,"steppedLine":false,"targets":[{"expr":"max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",cluster=~\"$cluster\", job=~\"$job\",instance=~\"$instance\"}) * 1e3","format":"time_series","legendFormat":"{{slice}}","legendLink":null}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Stage Duration","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ms","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":false}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Query","titleSize":"h6"}],"schemaVersion":14,"style":"dark","tags":["prometheus-mixin"],"templating":{"list":[{"current":{"text":"default","value":"default"},"hide":0,"label":"Data source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":".+","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","hide":2,"includeAll":true,"label":"cluster","multi":true,"name":"cluster","options":[],"query":"label_values(prometheus_build_info{job=\"prometheus-k8s\",namespace=\"monitoring\"}, cluster)","refresh":1,"regex":"","sort":2,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","hide":0,"includeAll":true,"label":"job","multi":true,"name":"job","options":[],"query":"label_values(prometheus_build_info{cluster=~\"$cluster\"}, job)","refresh":1,"regex":"","sort":2,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","current":{"selected":true,"text":"All","value":"$__all"},"datasource":"$datasource","hide":0,"includeAll":true,"label":"instance","multi":true,"name":"instance","options":[],"query":"label_values(prometheus_build_info{cluster=~\"$cluster\", job=~\"$job\"}, instance)","refresh":1,"regex":"","sort":2,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Prometheus / Overview","uid":"","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/scheduler.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-scheduler
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  scheduler.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"refresh":"10s","rows":[{"collapse":false,"collapsed":false,"panels":[{"cacheTimeout":null,"colorBackground":false,"colorValue":false,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"$datasource","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{},"id":2,"interval":"1m","legend":{"alignAsTable":true,"rightSide":true},"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"span":2,"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(up{cluster=\"$cluster\", job=\"kube-scheduler\"})","format":"time_series","intervalFactor":2,"legendFormat":"","refId":"A"}],"thresholds":"","title":"Up","tooltip":{"shared":false},"type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"min"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":3,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":5,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(scheduler_e2e_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} e2e","refId":"A"},{"expr":"sum(rate(scheduler_binding_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} binding","refId":"B"},{"expr":"sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} scheduling algorithm","refId":"C"},{"expr":"sum(rate(scheduler_volume_scheduling_duration_seconds_count{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance)","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} volume","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Scheduling Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":4,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":5,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} e2e","refId":"A"},{"expr":"histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} binding","refId":"B"},{"expr":"histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} scheduling algorithm","refId":"C"},{"expr":"histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}[$__rate_interval])) by (cluster, instance, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{cluster}} {{instance}} volume","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Scheduling latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":5,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"2xx","refId":"A"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"3xx","refId":"B"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"4xx","refId":"C"},{"expr":"sum(rate(rest_client_requests_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[$__rate_interval]))","format":"time_series","intervalFactor":2,"legendFormat":"5xx","refId":"D"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Kube API Request Rate","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"ops","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":6,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":8,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[$__rate_interval])) by (verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Post Request Latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":7,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":false,"steppedLine":false,"targets":[{"expr":"histogram_quantile(0.99, sum(rate(rest_client_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[$__rate_interval])) by (verb, url, le))","format":"time_series","intervalFactor":2,"legendFormat":"{{verb}} {{url}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Get Request Latency 99th Quantile","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"s","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":8,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"process_resident_memory_bytes{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Memory","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":null,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":9,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(process_cpu_seconds_total{cluster=\"$cluster\", job=\"kube-scheduler\", instance=~\"$instance\"}[$__rate_interval])","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"CPU usage","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"bytes","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":1,"fillGradient":0,"gridPos":{},"id":10,"interval":"1m","legend":{"alignAsTable":true,"avg":false,"current":false,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":4,"stack":false,"steppedLine":false,"targets":[{"expr":"go_goroutines{cluster=\"$cluster\", job=\"kube-scheduler\",instance=~\"$instance\"}","format":"time_series","intervalFactor":2,"legendFormat":"{{instance}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Goroutines","tooltip":{"shared":false,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":false,"title":"Dashboard Row","titleSize":"h6","type":"row"}],"schemaVersion":14,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":"cluster","multi":false,"name":"cluster","options":[],"query":"label_values(up{job=\"kube-scheduler\"}, cluster)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"current":{},"datasource":"$datasource","hide":0,"includeAll":true,"label":null,"multi":false,"name":"instance","options":[],"query":"label_values(up{job=\"kube-scheduler\", cluster=\"$cluster\"}, instance)","refresh":2,"regex":"","sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Scheduler","uid":"2e6b6a3b4bddf1427b3a55aa1311c656","version":0}
---
# Source: mimir/charts/kps/templates/grafana/dashboards-1.14/workload-total.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: default
  name: release-name-kps-workload-total
  annotations:
    {}
  labels:
    grafana_dashboard: "1"
    app: kps-grafana
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
data:
  workload-total.json: |-
    {"__inputs":[],"__requires":[],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":2,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Current Bandwidth","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":1},"id":3,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ pod }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":1},"id":4,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ pod }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Current Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":10},"id":5,"panels":[{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":11},"id":6,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ pod }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Received","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":true,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":11},"id":7,"legend":{"alignAsTable":true,"avg":false,"current":true,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":true,"show":true,"sideWidth":null,"sort":"current","sortDesc":true,"total":false,"values":true},"lines":false,"linewidth":1,"links":[],"minSpan":24,"nullPointMode":"null","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":24,"stack":false,"steppedLine":false,"targets":[{"expr":"sort_desc(avg(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{ pod }}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Average Rate of Bytes Transmitted","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"series","name":null,"show":false,"values":["current"]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Average Bandwidth","titleSize":"h6","type":"row"},{"collapse":false,"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":11},"id":8,"panels":[],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Bandwidth HIstory","titleSize":"h6","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":12},"id":9,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Receive Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":12},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_bytes_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Transmit Bandwidth","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"Bps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":11,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":22},"id":12,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":22},"id":13,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Packets","titleSize":"h6","type":"row"},{"collapse":true,"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":14,"panels":[{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":0,"y":23},"id":15,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_receive_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Received Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"$datasource","fill":2,"fillGradient":0,"gridPos":{"h":9,"w":12,"x":12,"y":23},"id":16,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":true,"hideZero":true,"max":false,"min":false,"rightSide":false,"show":true,"sideWidth":null,"total":false,"values":false},"lines":true,"linewidth":2,"links":[],"minSpan":12,"nullPointMode":"connected","paceLength":10,"percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"span":12,"stack":true,"steppedLine":false,"targets":[{"expr":"sort_desc(sum(irate(container_network_transmit_packets_dropped_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\",namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\", workload_type=~\"$type\"}) by (pod))\n","format":"time_series","intervalFactor":1,"legendFormat":"{{pod}}","refId":"A","step":10}],"thresholds":[],"timeFrom":null,"timeShift":null,"title":"Rate of Transmitted Packets Dropped","tooltip":{"shared":true,"sort":2,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true},{"format":"pps","label":null,"logBase":1,"max":null,"min":0,"show":true}]}],"repeat":null,"repeatIteration":null,"repeatRowId":null,"showTitle":true,"title":"Errors","titleSize":"h6","type":"row"}],"refresh":"10s","rows":[],"schemaVersion":18,"style":"dark","tags":["kubernetes-mixin"],"templating":{"list":[{"current":{"selected":true,"text":"default","value":"default"},"hide":0,"label":"Data Source","name":"datasource","options":[],"query":"prometheus","refresh":1,"regex":"","type":"datasource"},{"allValue":null,"current":{},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"cluster","options":[],"query":"label_values(kube_pod_info{job=\"kube-state-metrics\"}, cluster)","refresh":2,"regex":"","sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":".+","auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"kube-system","value":"kube-system"},"datasource":"$datasource","definition":"label_values(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"namespace","options":[],"query":"label_values(container_network_receive_packets_total{job=\"kubelet\", metrics_path=\"/metrics/cadvisor\", cluster=\"$cluster\"}, namespace)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)","hide":0,"includeAll":false,"label":null,"multi":false,"name":"workload","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\"}, workload)","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"","value":""},"datasource":"$datasource","definition":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)","hide":0,"includeAll":true,"label":null,"multi":false,"name":"type","options":[],"query":"label_values(namespace_workload_pod:kube_pod_owner:relabel{cluster=\"$cluster\",namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)","refresh":2,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":0,"includeAll":false,"label":null,"multi":false,"name":"resolution","options":[{"selected":false,"text":"30s","value":"30s"},{"selected":true,"text":"5m","value":"5m"},{"selected":false,"text":"1h","value":"1h"}],"query":"30s,5m,1h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false},{"allValue":null,"auto":false,"auto_count":30,"auto_min":"10s","current":{"text":"5m","value":"5m"},"datasource":"$datasource","hide":2,"includeAll":false,"label":null,"multi":false,"name":"interval","options":[{"selected":true,"text":"4h","value":"4h"}],"query":"4h","refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"interval","useTags":false}]},"time":{"from":"now-1h","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone": "utc","title":"Kubernetes / Networking / Workload","uid":"728bf77cc1166d2f3133bf25846876cc","version":0}
---
# Source: mimir/charts/mimir/charts/minio/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-minio
  labels:
    app: minio
    chart: minio-5.0.14
    release: release-name
    heritage: Helm
data:
  initialize: |-
    #!/bin/sh
    set -e ; # Have script exit in the event of a failed command.
    MC_CONFIG_DIR="/tmp/minio/mc/"
    MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
    
    # connectToMinio
    # Use a check-sleep-check loop to wait for MinIO service to be available
    connectToMinio() {
      SCHEME=$1
      ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
      set -e ; # fail if we can't read the keys.
      ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
      set +e ; # The connections to minio are allowed to fail.
      echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
      MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
      $MC_COMMAND ;
      STATUS=$? ;
      until [ $STATUS = 0 ]
      do
        ATTEMPTS=`expr $ATTEMPTS + 1` ;
        echo \"Failed attempts: $ATTEMPTS\" ;
        if [ $ATTEMPTS -gt $LIMIT ]; then
          exit 1 ;
        fi ;
        sleep 2 ; # 1 second intervals between attempts
        $MC_COMMAND ;
        STATUS=$? ;
      done ;
      set -e ; # reset `e` as active
      return 0
    }
    
    # checkBucketExists ($bucket)
    # Check if the bucket exists, by using the exit code of `mc ls`
    checkBucketExists() {
      BUCKET=$1
      CMD=$(${MC} stat myminio/$BUCKET > /dev/null 2>&1)
      return $?
    }
    
    # createBucket ($bucket, $policy, $purge)
    # Ensure bucket exists, purging if asked to
    createBucket() {
      BUCKET=$1
      POLICY=$2
      PURGE=$3
      VERSIONING=$4
      OBJECTLOCKING=$5
    
      # Purge the bucket, if set & exists
      # Since PURGE is user input, check explicitly for `true`
      if [ $PURGE = true ]; then
        if checkBucketExists $BUCKET ; then
          echo "Purging bucket '$BUCKET'."
          set +e ; # don't exit if this fails
          ${MC} rm -r --force myminio/$BUCKET
          set -e ; # reset `e` as active
        else
          echo "Bucket '$BUCKET' does not exist, skipping purge."
        fi
      fi
    
    # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created)
    if ! checkBucketExists $BUCKET ; then
        if [ ! -z $OBJECTLOCKING ] ; then
          if [ $OBJECTLOCKING = true ] ; then
              echo "Creating bucket with OBJECTLOCKING '$BUCKET'"
              ${MC} mb --with-lock myminio/$BUCKET
          elif [ $OBJECTLOCKING = false ] ; then
                echo "Creating bucket '$BUCKET'"
                ${MC} mb myminio/$BUCKET
          fi
      elif [ -z $OBJECTLOCKING ] ; then
            echo "Creating bucket '$BUCKET'"
            ${MC} mb myminio/$BUCKET
      else
        echo "Bucket '$BUCKET' already exists."  
      fi
      fi
    
    
      # set versioning for bucket if objectlocking is disabled or not set
      if [ $OBJECTLOCKING = false ] ; then
      if [ ! -z $VERSIONING ] ; then
        if [ $VERSIONING = true ] ; then
            echo "Enabling versioning for '$BUCKET'"
            ${MC} version enable myminio/$BUCKET
        elif [ $VERSIONING = false ] ; then
            echo "Suspending versioning for '$BUCKET'"
            ${MC} version suspend myminio/$BUCKET
        fi
        fi
      else
          echo "Bucket '$BUCKET' versioning unchanged."
      fi
    
    
      # At this point, the bucket should exist, skip checking for existence
      # Set policy on the bucket
      echo "Setting policy of bucket '$BUCKET' to '$POLICY'."
      ${MC} anonymous set $POLICY myminio/$BUCKET
    }
    
    # Try connecting to MinIO instance
    scheme=http
    connectToMinio $scheme
    
    
    
    # Create the buckets
    createBucket mimir-tsdb "none" false false false
    createBucket mimir-ruler "none" false false false
    createBucket enterprise-metrics-tsdb "none" false false false
    createBucket enterprise-metrics-admin "none" false false false
    createBucket enterprise-metrics-ruler "none" false false false
    
  add-user: |-
    #!/bin/sh
    set -e ; # Have script exit in the event of a failed command.
    MC_CONFIG_DIR="/tmp/minio/mc/"
    MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
    
    # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters.
    # Special characters for example : ',",<,>,{,}
    MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp"
    
    # connectToMinio
    # Use a check-sleep-check loop to wait for MinIO service to be available
    connectToMinio() {
      SCHEME=$1
      ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
      set -e ; # fail if we can't read the keys.
      ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
      set +e ; # The connections to minio are allowed to fail.
      echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
      MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
      $MC_COMMAND ;
      STATUS=$? ;
      until [ $STATUS = 0 ]
      do
        ATTEMPTS=`expr $ATTEMPTS + 1` ;
        echo \"Failed attempts: $ATTEMPTS\" ;
        if [ $ATTEMPTS -gt $LIMIT ]; then
          exit 1 ;
        fi ;
        sleep 2 ; # 1 second intervals between attempts
        $MC_COMMAND ;
        STATUS=$? ;
      done ;
      set -e ; # reset `e` as active
      return 0
    }
    
    # checkUserExists ()
    # Check if the user exists, by using the exit code of `mc admin user info`
    checkUserExists() {
      CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1)
      return $?
    }
    
    # createUser ($policy)
    createUser() {
      POLICY=$1
      #check accessKey_and_secretKey_tmp file
      if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then
        echo "credentials file does not exist"
        return 1
      fi
      if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then
        echo "credentials file is invalid"
        rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
        return 1
      fi
      USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP)
      # Create the user if it does not exist
      if ! checkUserExists ; then
        echo "Creating user '$USER'"
        cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio
      else
        echo "User '$USER' already exists."
      fi
      #clean up credentials files.
      rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
    
      # set policy for user
      if [ ! -z $POLICY -a $POLICY != " " ] ; then
          echo "Adding policy '$POLICY' for '$USER'"
          set +e ; # policy already attach errors out, allow it.
          ${MC} admin policy attach myminio $POLICY --user=$USER
          set -e
      else
          echo "User '$USER' has no policy attached."
      fi
    }
    
    # Try connecting to MinIO instance
    scheme=http
    connectToMinio $scheme
    
    
    
    # Create the users
    echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP
    echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP
    createUser consoleAdmin
    
  add-policy: |-
    #!/bin/sh
    set -e ; # Have script exit in the event of a failed command.
    MC_CONFIG_DIR="/tmp/minio/mc/"
    MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
    
    # connectToMinio
    # Use a check-sleep-check loop to wait for MinIO service to be available
    connectToMinio() {
      SCHEME=$1
      ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
      set -e ; # fail if we can't read the keys.
      ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
      set +e ; # The connections to minio are allowed to fail.
      echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
      MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
      $MC_COMMAND ;
      STATUS=$? ;
      until [ $STATUS = 0 ]
      do
        ATTEMPTS=`expr $ATTEMPTS + 1` ;
        echo \"Failed attempts: $ATTEMPTS\" ;
        if [ $ATTEMPTS -gt $LIMIT ]; then
          exit 1 ;
        fi ;
        sleep 2 ; # 1 second intervals between attempts
        $MC_COMMAND ;
        STATUS=$? ;
      done ;
      set -e ; # reset `e` as active
      return 0
    }
    
    # checkPolicyExists ($policy)
    # Check if the policy exists, by using the exit code of `mc admin policy info`
    checkPolicyExists() {
      POLICY=$1
      CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1)
      return $?
    }
    
    # createPolicy($name, $filename)
    createPolicy () {
      NAME=$1
      FILENAME=$2
    
      # Create the name if it does not exist
      echo "Checking policy: $NAME (in /config/$FILENAME.json)"
      if ! checkPolicyExists $NAME ; then
        echo "Creating policy '$NAME'"
      else
        echo "Policy '$NAME' already exists."
      fi
      ${MC} admin policy create myminio $NAME /config/$FILENAME.json
    
    }
    
    # Try connecting to MinIO instance
    scheme=http
    connectToMinio $scheme
    
    
    
  add-svcacct: |-
    #!/bin/sh
    set -e ; # Have script exit in the event of a failed command.
    MC_CONFIG_DIR="/tmp/minio/mc/"
    MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
    
    # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters.
    # Special characters for example : ',",<,>,{,}
    MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp"
    
    # connectToMinio
    # Use a check-sleep-check loop to wait for MinIO service to be available
    connectToMinio() {
      SCHEME=$1
      ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
      set -e ; # fail if we can't read the keys.
      ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
      set +e ; # The connections to minio are allowed to fail.
      echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
      MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
      $MC_COMMAND ;
      STATUS=$? ;
      until [ $STATUS = 0 ]
      do
        ATTEMPTS=`expr $ATTEMPTS + 1` ;
        echo \"Failed attempts: $ATTEMPTS\" ;
        if [ $ATTEMPTS -gt $LIMIT ]; then
          exit 1 ;
        fi ;
        sleep 2 ; # 2 second intervals between attempts
        $MC_COMMAND ;
        STATUS=$? ;
      done ;
      set -e ; # reset `e` as active
      return 0
    }
    
    # checkSvcacctExists ()
    # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info`
    checkSvcacctExists() {
      CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1)
      return $?
    }
    
    # createSvcacct ($user)
    createSvcacct () {
      USER=$1
      FILENAME=$2
      #check accessKey_and_secretKey_tmp file
      if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then
        echo "credentials file does not exist"
        return 1
      fi
      if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then
        echo "credentials file is invalid"
        rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
        return 1
      fi
      SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP)
      # Create the svcacct if it does not exist
      if ! checkSvcacctExists ; then
        echo "Creating svcacct '$SVCACCT'"
        # Check if policy file is define
        if [ -z $FILENAME ]; then
          ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER
        else
          ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER
        fi
      else
        echo "Svcacct '$SVCACCT' already exists."
      fi
      #clean up credentials files.
      rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP
    }
    
    # Try connecting to MinIO instance
    scheme=http
    connectToMinio $scheme
    
    
    
  custom-command: |-
    #!/bin/sh
    set -e ; # Have script exit in the event of a failed command.
    MC_CONFIG_DIR="/tmp/minio/mc/"
    MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}"
    
    # connectToMinio
    # Use a check-sleep-check loop to wait for MinIO service to be available
    connectToMinio() {
      SCHEME=$1
      ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
      set -e ; # fail if we can't read the keys.
      ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
      set +e ; # The connections to minio are allowed to fail.
      echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
      MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
      $MC_COMMAND ;
      STATUS=$? ;
      until [ $STATUS = 0 ]
      do
        ATTEMPTS=`expr $ATTEMPTS + 1` ;
        echo \"Failed attempts: $ATTEMPTS\" ;
        if [ $ATTEMPTS -gt $LIMIT ]; then
          exit 1 ;
        fi ;
        sleep 2 ; # 1 second intervals between attempts
        $MC_COMMAND ;
        STATUS=$? ;
      done ;
      set -e ; # reset `e` as active
      return 0
    }
    
    # runCommand ($@)
    # Run custom mc command
    runCommand() {
      ${MC} "$@"
      return $?
    }
    
    # Try connecting to MinIO instance
    scheme=http
    connectToMinio $scheme
---
# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-mimir-alertmanager-fallback-config
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: alertmanager
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {} 
  namespace: "default"
data:
  alertmanager_fallback_config.yaml: |
    receivers:
        - name: default-receiver
    route:
        receiver: default-receiver
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-alertmanager-resources-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-alertmanager-resources.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Alertmanager",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?alertmanager.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?alertmanager.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Network",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_written_bytes_total[$__rate_interval]\n  )\n)\n+\nignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"alertmanager\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk writes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_read_bytes_total[$__rate_interval]\n  )\n) + ignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"alertmanager\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk reads",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Disk",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(alertmanager).*\"\n  }\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{persistentvolumeclaim}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk space utilization",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Alertmanager resources",
          "uid": "a6883fb22799ac74479c7db872451092",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-alertmanager-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-alertmanager.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "100px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "format": "short",
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_job_pod:cortex_alertmanager_alerts:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Total alerts",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "format": "short",
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_job_pod:cortex_alertmanager_silences:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Total silences",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "format": "short",
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max(cortex_alertmanager_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Tenants",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Headlines",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "QPS",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", route=~\"/alertmanagerpb.Alertmanager/HandleRequest\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Alertmanager Distributor",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_job:cortex_alertmanager_alerts_received_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "APS",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Alerts received",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "cortex_alertmanager_dispatcher_aggregation_groups{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "per pod Active Aggregation Groups",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Alerts grouping",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "NPS",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "(\nsum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)\n) > 0\nor on () vector(0)\n",
                            "format": "time_series",
                            "legendFormat": "success - {{ integration }}",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) by(integration)",
                            "format": "time_series",
                            "legendFormat": "failed - {{ integration }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "NPS by integration",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Alert notifications",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Operations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "max": 1,
                            "min": 0,
                            "noValue": "0",
                            "unit": "percentunit"
                         }
                      },
                      "id": 12,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 3,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\"}[$__rate_interval])) >= 0",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Error rate",
                      "type": "timeseries"
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Attributes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Exists",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Alertmanager Configuration Object Store (Alertmanager accesses)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Get",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 16,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: GetRange",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Upload",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 18,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Delete",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (cortex_alertmanager_tenants_owned{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Per pod tenants",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 20,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_alerts:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Per pod alerts",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_silences:sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Per pod silences",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Replication",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 22,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_alertmanager_sync_configs_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_sync_configs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_alertmanager_sync_configs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Syncs/sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 23,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(reason) (rate(cortex_alertmanager_sync_configs_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{reason}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Syncs/sec (by reason)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 24,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum (rate(cortex_alertmanager_ring_check_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "errors",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Ring check errors/sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Tenant configuration sync",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 25,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(outcome) (rate(cortex_alertmanager_state_initial_sync_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "interval": "1m",
                            "legendFormat": "{{outcome}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Initial syncs /sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 26,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "interval": "1m",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "interval": "1m",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "interval": "1m",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Initial sync duration",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 27,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "interval": "1m",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "interval": "1m",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Fetch state from other alertmanagers /sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Sharding initial state sync",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 28,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_job:cortex_alertmanager_state_replication_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Replicate state to other alertmanagers /sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 29,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Merge state from other alertmanagers /sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 30,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_alertmanager_state_persist_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_persist_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_alertmanager_state_persist_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Persist state to remote storage /sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Sharding runtime state sync",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Alertmanager",
          "uid": "b0d38d318bbddd80476246d4930f9e55",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-compactor-resources-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-compactor-resources.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "CPU and memory",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (RSS)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"compactor\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?compactor.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?compactor.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Network",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_written_bytes_total[$__rate_interval]\n  )\n)\n+\nignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"compactor\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk writes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_read_bytes_total[$__rate_interval]\n  )\n) + ignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"compactor\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk reads",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(compactor).*\"\n  }\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{persistentvolumeclaim}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk space utilization",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Disk",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Compactor resources",
          "uid": "09a5c49e9cdb2f2b24c6d184574a07fd",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-compactor-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-compactor.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "completed": "#7EB26D",
                         "failed": "#E24D42",
                         "started": "#34CCEB"
                      },
                      "bars": true,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n",
                      "fill": 10,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": false,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_compactor_runs_started_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "started",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_compactor_runs_completed_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "completed",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_compactor_runs_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Per-instance runs / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\n\n",
                      "fieldConfig": {
                         "defaults": {
                            "max": 1,
                            "noValue": 1,
                            "unit": "percentunit"
                         }
                      },
                      "id": 2,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 3,
                      "targets": [
                         {
                            "expr": "(\n  cortex_compactor_tenants_processing_succeeded{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} +\n  cortex_compactor_tenants_processing_failed{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} +\n  cortex_compactor_tenants_skipped{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}\n)\n/\ncortex_compactor_tenants_discovered{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"} > 0\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Tenants compaction progress",
                      "type": "timeseries"
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Longest time since last successful run\nDisplays the amount of time since the most recent successful execution\nof the compactor.\nThe value shown will be for the compactor replica that has the longest time since its\nlast successful run.\nThe table to the right shows a summary for all compactor replicas.\n\nIf there is no time value, one of the following messages might appear:\n\n- If you see \"No compactor data\" in this panel, that means that no compactors are active yet.\n\n- If you see \"No successful runs\" in this panel, that means that compactors are active, but none\n  of them were successfully executed yet.\n\nThese might be expected - for example, if you just recently restarted your compactors,\nthey might not have had a chance to complete their first compaction run.\nHowever, if these messages persist, you should check the health of your compactors.\n\n",
                      "fieldConfig": {
                         "defaults": {
                            "color": {
                               "mode": "thresholds"
                            },
                            "decimals": 1,
                            "noValue": "No compactor data",
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "Last run"
                               },
                               "properties": [
                                  {
                                     "id": "custom.width",
                                     "value": 74
                                  },
                                  {
                                     "id": "mappings",
                                     "value": [
                                        {
                                           "options": {
                                              "from": "-Infinity",
                                              "result": {
                                                 "color": "text",
                                                 "text": "No successful runs since startup yet"
                                              },
                                              "to": 0
                                           },
                                           "type": "range"
                                        }
                                     ]
                                  },
                                  {
                                     "id": "color",
                                     "value": {
                                        "mode": "thresholds"
                                     }
                                  },
                                  {
                                     "id": "thresholds",
                                     "value": {
                                        "mode": "absolute",
                                        "steps": [
                                           {
                                              "color": "green",
                                              "value": 0
                                           },
                                           {
                                              "color": "yellow",
                                              "value": 7200
                                           },
                                           {
                                              "color": "orange",
                                              "value": 21600
                                           },
                                           {
                                              "color": "red",
                                              "value": 43200
                                           }
                                        ]
                                     }
                                  }
                               ]
                            }
                         ]
                      },
                      "fill": 1,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "options": {
                         "reduceOptions": {
                            "calcs": [
                               "first"
                            ],
                            "fields": "/^Last run$/",
                            "values": false
                         },
                         "textMode": "value"
                      },
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod)\n(\n  (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h]) !=bool 0))\n  -\n  max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h])\n)\n",
                            "format": "table",
                            "instant": true,
                            "interval": "",
                            "intervalFactor": null,
                            "legendFormat": "",
                            "legendLink": null,
                            "step": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Longest time since last successful run",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transformations": [
                         {
                            "id": "organize",
                            "options": {
                               "renameByName": {
                                  "Value": "Last run",
                                  "pod": "Compactor"
                               }
                            }
                         },
                         {
                            "id": "sortBy",
                            "options": {
                               "sort": [
                                  {
                                     "desc": true,
                                     "field": "Last run"
                                  }
                               ]
                            }
                         }
                      ],
                      "type": "stat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Last successful run per-compactor replica\nDisplays the compactor replicas, and for each, shows how long it has been since\nits last successful compaction run.\n\nThe value in the status column is based on how long it has been since the last successful compaction.\n\n- Okay: less than 2 hours\n- Delayed: more than 2 hours\n- Late: more than 6 hours\n- Very late: more than 12 hours\n\nIf the status of any compactor replicas are *Late* or *Very late*, check their health.\n\n",
                      "fieldConfig": {
                         "overrides": [
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "Status"
                               },
                               "properties": [
                                  {
                                     "id": "custom.displayMode",
                                     "value": "color-background"
                                  },
                                  {
                                     "id": "mappings",
                                     "value": [
                                        {
                                           "options": {
                                              "from": "-Infinity",
                                              "result": {
                                                 "color": "transparent",
                                                 "text": "N/A"
                                              },
                                              "to": 0
                                           },
                                           "type": "range"
                                        },
                                        {
                                           "options": {
                                              "from": 0,
                                              "result": {
                                                 "color": "green",
                                                 "text": "Ok"
                                              },
                                              "to": 7200
                                           },
                                           "type": "range"
                                        },
                                        {
                                           "options": {
                                              "from": 7200,
                                              "result": {
                                                 "color": "yellow",
                                                 "text": "Delayed"
                                              },
                                              "to": 21600
                                           },
                                           "type": "range"
                                        },
                                        {
                                           "options": {
                                              "from": 21600,
                                              "result": {
                                                 "color": "orange",
                                                 "text": "Late"
                                              },
                                              "to": 43200
                                           },
                                           "type": "range"
                                        },
                                        {
                                           "options": {
                                              "from": 43200,
                                              "result": {
                                                 "color": "red",
                                                 "text": "Very late"
                                              },
                                              "to": "Infinity"
                                           },
                                           "type": "range"
                                        },
                                        {
                                           "options": {
                                              "match": "null+nan",
                                              "result": {
                                                 "color": "transparent",
                                                 "text": "Unknown"
                                              }
                                           },
                                           "type": "special"
                                        }
                                     ]
                                  },
                                  {
                                     "id": "custom.width",
                                     "value": 86
                                  },
                                  {
                                     "id": "custom.align",
                                     "value": "center"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "Last run"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "s"
                                  },
                                  {
                                     "id": "custom.width",
                                     "value": 74
                                  },
                                  {
                                     "id": "mappings",
                                     "value": [
                                        {
                                           "options": {
                                              "from": "-Infinity",
                                              "result": {
                                                 "text": "Never"
                                              },
                                              "to": 0
                                           },
                                           "type": "range"
                                        }
                                     ]
                                  }
                               ]
                            }
                         ]
                      },
                      "fill": 1,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod)\n(\n  (time() * (max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h]) !=bool 0))\n  -\n  max_over_time(cortex_compactor_last_successful_run_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[1h])\n)\n",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "Last run",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Last successful run per-compactor replica",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transformations": [
                         {
                            "id": "organize",
                            "options": {
                               "renameByName": {
                                  "Value": "Last run",
                                  "pod": "Compactor"
                               }
                            }
                         },
                         {
                            "id": "sortBy",
                            "options": {
                               "sort": [
                                  {
                                     "desc": true,
                                     "field": "Last run"
                                  }
                               ]
                            }
                         },
                         {
                            "id": "calculateField",
                            "options": {
                               "alias": "One",
                               "binary": {
                                  "left": "Last run",
                                  "operator": "/",
                                  "right": "Last run"
                               },
                               "mode": "binary",
                               "replaceFields": false
                            }
                         },
                         {
                            "id": "calculateField",
                            "options": {
                               "alias": "Status",
                               "binary": {
                                  "left": "Last run",
                                  "operator": "*",
                                  "right": "One"
                               },
                               "mode": "binary",
                               "replaceFields": false
                            }
                         },
                         {
                            "id": "filterFieldsByName",
                            "options": {
                               "include": {
                                  "names": [
                                     "Compactor",
                                     "Last run",
                                     "Status"
                                  ]
                               }
                            }
                         }
                      ],
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Summary",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### TSDB compactions / sec\nRate of TSDB compactions. Single TSDB compaction takes one or more input blocks and produces one or more (during \"split\" phase) output blocks.\n\n",
                      "fill": 1,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(prometheus_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "compactions",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TSDB compactions / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### TSDB compaction duration\nDisplay the amount of time that it has taken to run a single TSDB compaction.\n\n",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TSDB compaction duration",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Average blocks / tenant",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}))",
                            "format": "time_series",
                            "legendFormat": "{{user}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Tenants with largest number of blocks",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "blocks",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Blocks marked for deletion / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Blocks deletions / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Garbage collector",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_compactor_meta_syncs_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Metadata syncs / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Metadata sync duration",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Metadata sync",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Operations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "max": 1,
                            "min": 0,
                            "noValue": "0",
                            "unit": "percentunit"
                         }
                      },
                      "id": 14,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 3,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\"}[$__rate_interval])) >= 0",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Error rate",
                      "type": "timeseries"
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Attributes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 16,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Exists",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Object Store",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Get",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 18,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: GetRange",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Upload",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 20,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Delete",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 22,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\", kv_name=~\".+\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Key-value store for compactors ring",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Compactor",
          "uid": "1b3443aea86db629e6efdb7d05c53823",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-config-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-config.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "count(cortex_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
                            "format": "time_series",
                            "legendFormat": "sha256:{{sha256}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Startup config file hashes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "instances",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Startup config file",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "count(cortex_runtime_config_hash{cluster=~\"$cluster\", namespace=~\"$namespace\"}) by (sha256)",
                            "format": "time_series",
                            "legendFormat": "sha256:{{sha256}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Runtime config file hashes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "instances",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Runtime config file",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Config",
          "uid": "5d9d0b4724c0f80d68467088ec61e003",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-object-store-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-object-store.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{component}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "RPS / component",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "max": 1,
                            "min": 0,
                            "noValue": "0",
                            "unit": "percentunit"
                         }
                      },
                      "id": 2,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 6,
                      "targets": [
                         {
                            "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >= 0",
                            "format": "time_series",
                            "legendFormat": "{{component}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Error rate / component",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Components",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "RPS / operation",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "max": 1,
                            "min": 0,
                            "noValue": "0",
                            "unit": "percentunit"
                         }
                      },
                      "id": 4,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 6,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >= 0",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Error rate / operation",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Operations",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Op: Get",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Op: GetRange",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Op: Exists",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Op: Attributes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Op: Upload",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Op: Delete",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Object Store",
          "uid": "e1324ee2a434f4158c00a9ee279d3292",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-overrides-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-overrides.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "datasource": "${datasource}",
                      "id": 1,
                      "span": 12,
                      "targets": [
                         {
                            "expr": "max by(limit_name) (cortex_limits_defaults{cluster=~\"$cluster\",namespace=~\"$namespace\"})",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "title": "Defaults",
                      "transformations": [
                         {
                            "id": "labelsToFields",
                            "options": { }
                         },
                         {
                            "id": "merge",
                            "options": { }
                         },
                         {
                            "id": "organize",
                            "options": {
                               "excludeByName": {
                                  "Time": true
                               },
                               "indexByName": {
                                  "Value": 1,
                                  "limit_name": 0
                               }
                            }
                         },
                         {
                            "id": "sortBy",
                            "options": {
                               "fields": { },
                               "sort": [
                                  {
                                     "field": "limit_name"
                                  }
                               ]
                            }
                         }
                      ],
                      "type": "table"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "datasource": "${datasource}",
                      "id": 2,
                      "span": 12,
                      "targets": [
                         {
                            "expr": "max by(user, limit_name) (cortex_limits_overrides{cluster=~\"$cluster\",namespace=~\"$namespace\",user=~\"${tenant_id}\"})",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "title": "Per-tenant overrides",
                      "transformations": [
                         {
                            "id": "labelsToFields",
                            "options": {
                               "mode": "columns",
                               "valueLabel": "limit_name"
                            }
                         },
                         {
                            "id": "merge",
                            "options": { }
                         },
                         {
                            "id": "organize",
                            "options": {
                               "excludeByName": {
                                  "Time": true
                               },
                               "indexByName": {
                                  "user": 0
                               }
                            }
                         }
                      ],
                      "type": "table"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "current": {
                      "selected": true,
                      "text": ".*",
                      "value": ".*"
                   },
                   "hide": 0,
                   "label": "Tenant ID",
                   "name": "tenant_id",
                   "options": [
                      {
                         "selected": true,
                         "text": ".*",
                         "value": ".*"
                      }
                   ],
                   "query": ".*",
                   "type": "textbox"
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Overrides",
          "uid": "1e2c358600ac53f09faea133f811b5bb",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-overview-networking-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-overview-networking.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Writes",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Reads",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Backend",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Overview networking",
          "uid": "e15c71d372cc541367a088f10d9fcd92",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-overview-resources-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-overview-resources.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Writes",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_written_bytes_total[$__rate_interval]\n  )\n)\n+\nignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"distributor|ingester|mimir-write\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk writes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_read_bytes_total[$__rate_interval]\n  )\n) + ignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"distributor|ingester|mimir-write\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk reads",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(distributor|ingester|mimir-write).*\"\n  }\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{persistentvolumeclaim}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk space utilization",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Reads",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Backend",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_written_bytes_total[$__rate_interval]\n  )\n)\n+\nignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk writes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_read_bytes_total[$__rate_interval]\n  )\n) + ignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk reads",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(query-scheduler|ruler-query-scheduler|ruler|store-gateway|compactor|alertmanager|overrides-exporter|mimir-backend).*\"\n  }\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{persistentvolumeclaim}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk space utilization",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Overview resources",
          "uid": "a9b92d3c4d1af325d872a9e9a7083d71",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-overview-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-overview.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "content": "The 'Status' panel shows an overview on the cluster health over the time.\nTo investigate failures, see a specific dashboard:\n\n- <a target=\"_blank\" href=\"./d/8280707b8f16e7b87b840fc1cc92d4c5/mimir-writes?${__url_time_range}&${__all_variables}\">Writes</a>\n- <a target=\"_blank\" href=\"./d/e327503188913dc38ad571c647eef643/mimir-reads?${__url_time_range}&${__all_variables}\">Reads</a>\n- <a target=\"_blank\" href=\"./d/631e15d5d85afb2ca8e35d62984eeaa0/mimir-ruler?${__url_time_range}&${__all_variables}\">Rule evaluations</a>\n- <a target=\"_blank\" href=\"./d/b0d38d318bbddd80476246d4930f9e55/mimir-alertmanager?${__url_time_range}&${__all_variables}\">Alerting notifications</a>\n- <a target=\"_blank\" href=\"./d/e1324ee2a434f4158c00a9ee279d3292/mimir-object-store?${__url_time_range}&${__all_variables}\">Object storage</a>\n",
                      "datasource": null,
                      "description": "",
                      "id": 1,
                      "mode": "markdown",
                      "span": 3,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   },
                   {
                      "fieldConfig": {
                         "defaults": {
                            "color": {
                               "mode": "thresholds"
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [
                                  {
                                     "color": "#7EB26D",
                                     "value": null
                                  },
                                  {
                                     "color": "#EAB839",
                                     "value": 0.01
                                  },
                                  {
                                     "color": "#E24D42",
                                     "value": 0.050000000000000003
                                  }
                               ]
                            }
                         }
                      },
                      "id": 2,
                      "options": {
                         "showValue": "never"
                      },
                      "span": 6,
                      "targets": [
                         {
                            "datasource": {
                               "uid": "$datasource"
                            },
                            "exemplar": false,
                            "expr": "(\n    # gRPC errors are not tracked as 5xx but \"error\".\n    sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"5.*|error\"}[$__rate_interval]))\n    or\n    # Handle the case no failure has been tracked yet.\n    vector(0)\n)\n/\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
                            "instant": false,
                            "legendFormat": "Writes",
                            "range": true
                         },
                         {
                            "datasource": {
                               "uid": "$datasource"
                            },
                            "exemplar": false,
                            "expr": "(\n    sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.*\"}[$__rate_interval]))\n    or\n    # Handle the case no failure has been tracked yet.\n    vector(0)\n)\n/\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
                            "instant": false,
                            "legendFormat": "Reads",
                            "range": true
                         },
                         {
                            "datasource": {
                               "uid": "$datasource"
                            },
                            "exemplar": false,
                            "expr": "(\n  (\n      sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n      +\n      # Consider missed evaluations as failures.\n      sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n  )\n  or\n  # Handle the case no failure has been tracked yet.\n  vector(0)\n)\n/\nsum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "instant": false,
                            "legendFormat": "Rule evaluations",
                            "range": true
                         },
                         {
                            "datasource": {
                               "uid": "$datasource"
                            },
                            "exemplar": false,
                            "expr": "(\n  # Failed notifications from ruler to Alertmanager (handling the case the ruler metrics are missing).\n  ((sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n) or vector(0))\n  +\n  # Failed notifications from Alertmanager to receivers (handling the case the alertmanager metrics are missing).\n  ((sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n) or vector(0))\n)\n/\n(\n  # Total notifications from ruler to Alertmanager (handling the case the ruler metrics are missing).\n  ((sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n) or vector(0))\n  +\n  # Total notifications from Alertmanager to receivers (handling the case the alertmanager metrics are missing).\n  ((sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n) or vector(0))\n)\n",
                            "instant": false,
                            "legendFormat": "Alerting notifications",
                            "range": true
                         },
                         {
                            "datasource": {
                               "uid": "$datasource"
                            },
                            "exemplar": false,
                            "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n/\nsum(rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
                            "instant": false,
                            "legendFormat": "Object storage",
                            "range": true
                         }
                      ],
                      "title": "Status",
                      "type": "state-timeline"
                   },
                   {
                      "id": 3,
                      "options": {
                         "alertInstanceLabelFilter": "cluster=~\"$cluster\", namespace=~\"$namespace\"",
                         "alertName": "Mimir",
                         "dashboardAlerts": false,
                         "maxItems": 100,
                         "sortOrder": 3,
                         "stateFilter": {
                            "error": true,
                            "firing": true,
                            "noData": false,
                            "normal": false,
                            "pending": false
                         }
                      },
                      "span": 3,
                      "title": "Firing alerts",
                      "type": "alertlist"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Mimir cluster health",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "content": "These panels show an overview on the write path. \nTo examine the write path in detail, see a specific dashboard:\n\n- <a target=\"_blank\" href=\"./d/8280707b8f16e7b87b840fc1cc92d4c5/mimir-writes?${__url_time_range}&${__all_variables}\">Writes</a>\n- <a target=\"_blank\" href=\"./d/bc9160e50b52e89e0e49c840fea3d379/mimir-writes-resources?${__url_time_range}&${__all_variables}\">Writes resources</a>\n- <a target=\"_blank\" href=\"./d/978c1cb452585c96697a238eaac7fe2d/mimir-writes-networking?${__url_time_range}&${__all_variables}\">Writes networking</a>\n- <a target=\"_blank\" href=\"./d/a9b92d3c4d1af325d872a9e9a7083d71/mimir-overview-resources?${__url_time_range}&${__all_variables}\">Overview resources</a>\n- <a target=\"_blank\" href=\"./d/e15c71d372cc541367a088f10d9fcd92/mimir-overview-networking?${__url_time_range}&${__all_variables}\">Overview networking</a>\n",
                      "datasource": null,
                      "description": "",
                      "id": 4,
                      "mode": "markdown",
                      "span": 3,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   },
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Write requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Write latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "legendFormat": "samples / sec",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "legendFormat": "exemplars / sec",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Ingestion / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "cps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Writes",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "content": "These panels show an overview on the read path. \nTo examine the read path in detail, see a specific dashboard:\n\n- <a target=\"_blank\" href=\"./d/e327503188913dc38ad571c647eef643/mimir-reads?${__url_time_range}&${__all_variables}\">Reads</a>\n- <a target=\"_blank\" href=\"./d/cc86fd5aa9301c6528986572ad974db9/mimir-reads-resources?${__url_time_range}&${__all_variables}\">Reads resources</a>\n- <a target=\"_blank\" href=\"./d/54b2a0a4748b3bd1aefa92ce5559a1c2/mimir-reads-networking?${__url_time_range}&${__all_variables}\">Reads networking</a>\n- <a target=\"_blank\" href=\"./d/a9b92d3c4d1af325d872a9e9a7083d71/mimir-overview-resources?${__url_time_range}&${__all_variables}\">Overview resources</a>\n- <a target=\"_blank\" href=\"./d/e15c71d372cc541367a088f10d9fcd92/mimir-overview-networking?${__url_time_range}&${__all_variables}\">Overview networking</a>\n- <a target=\"_blank\" href=\"./d/b3abe8d5c040395cc36615cb4334c92d/mimir-queries?${__url_time_range}&${__all_variables}\">Queries</a>\n- <a target=\"_blank\" href=\"./d/1b3443aea86db629e6efdb7d05c53823/mimir-compactor?${__url_time_range}&${__all_variables}\">Compactor</a>\n",
                      "datasource": null,
                      "description": "",
                      "id": 8,
                      "mode": "markdown",
                      "span": 3,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   },
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Read requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Read latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "instant queries",
                            "color": "#429D48"
                         },
                         {
                            "alias": "range queries",
                            "color": "#F1C731"
                         },
                         {
                            "alias": "\"label names\" queries",
                            "color": "#2A66CF"
                         },
                         {
                            "alias": "\"label values\" queries",
                            "color": "#9E44C1"
                         },
                         {
                            "alias": "series queries",
                            "color": "#FFAB57"
                         },
                         {
                            "alias": "remote read queries",
                            "color": "#C79424"
                         },
                         {
                            "alias": "metadata queries",
                            "color": "#84D586"
                         },
                         {
                            "alias": "exemplar queries",
                            "color": "#A1C4FC"
                         },
                         {
                            "alias": "other",
                            "color": "#C788DE"
                         }
                      ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "instant queries",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query_range\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "range queries",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_labels\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "\"label names\" queries",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_label_name_values\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "\"label values\" queries",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_series\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "series queries",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_read\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "remote read queries",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_metadata\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "metadata queries",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query_exemplars\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "exemplar queries",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_.*\",route!~\".*(query|query_range|label.*|series|read|metadata|query_exemplars)\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "other",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Queries / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Reads",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "content": "These panels show an overview on the recording and alerting rules evaluation.\nTo examine the rules evaluation and alerts notifications in detail, see a specific dashboard:\n\n- <a target=\"_blank\" href=\"./d/631e15d5d85afb2ca8e35d62984eeaa0/mimir-ruler?${__url_time_range}&${__all_variables}\">Ruler</a>\n- <a target=\"_blank\" href=\"./d/b0d38d318bbddd80476246d4930f9e55/mimir-alertmanager?${__url_time_range}&${__all_variables}\">Alertmanager</a>\n- <a target=\"_blank\" href=\"./d/a6883fb22799ac74479c7db872451092/mimir-alertmanager-resources?${__url_time_range}&${__all_variables}\">Alertmanager resources</a>\n- <a target=\"_blank\" href=\"./d/a9b92d3c4d1af325d872a9e9a7083d71/mimir-overview-resources?${__url_time_range}&${__all_variables}\">Overview resources</a>\n- <a target=\"_blank\" href=\"./d/e15c71d372cc541367a088f10d9fcd92/mimir-overview-networking?${__url_time_range}&${__all_variables}\">Overview networking</a>\n",
                      "datasource": null,
                      "description": "",
                      "id": 12,
                      "mode": "markdown",
                      "span": 3,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "success",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "missed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Rule evaluations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n  /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "average",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Rule evaluations latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n  -\nsum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Alerting notifications sent to Alertmanager / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Recording and alerting rules",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "content": "These panels show an overview on the long-term storage (object storage).\nTo examine the storage in detail, see a specific dashboard:\n\n- <a target=\"_blank\" href=\"./d/e1324ee2a434f4158c00a9ee279d3292/mimir-object-store?${__url_time_range}&${__all_variables}\">Object store</a>\n- <a target=\"_blank\" href=\"./d/1b3443aea86db629e6efdb7d05c53823/mimir-compactor?${__url_time_range}&${__all_variables}\">Compactor</a>\n",
                      "datasource": null,
                      "description": "",
                      "id": 16,
                      "mode": "markdown",
                      "span": 3,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n-\nsum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 18,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "attributes",
                            "color": "#429D48"
                         },
                         {
                            "alias": "delete",
                            "color": "#F1C731"
                         },
                         {
                            "alias": "exists",
                            "color": "#2A66CF"
                         },
                         {
                            "alias": "get",
                            "color": "#9E44C1"
                         },
                         {
                            "alias": "get_range",
                            "color": "#FFAB57"
                         },
                         {
                            "alias": "iter",
                            "color": "#C79424"
                         },
                         {
                            "alias": "upload",
                            "color": "#84D586"
                         }
                      ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Operations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(max by(user) (max_over_time(cortex_bucket_blocks_count{cluster=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[15m])))",
                            "format": "time_series",
                            "legendFormat": "blocks",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Total number of blocks in the storage",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Long-term storage (object storage)",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Overview",
          "uid": "ffcd83628d7d4b5a03d1cafd159e6c9c",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-queries-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-queries.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Queue duration",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_query_frontend_retries_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Retries",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Queue length (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "noValue": "0",
                            "unit": "short"
                         }
                      },
                      "id": 4,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 3,
                      "targets": [
                         {
                            "expr": "sum by(user) (cortex_query_frontend_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}) > 0",
                            "format": "time_series",
                            "legendFormat": "{{user}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Queue length (per user)",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-frontend",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Queue duration",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Queue length (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "noValue": "0",
                            "unit": "short"
                         }
                      },
                      "id": 7,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "expr": "sum by(user) (cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}) > 0",
                            "format": "time_series",
                            "legendFormat": "{{user}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Queue length (per user)",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-scheduler",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Intervals per query\nThe average number of split queries (partitioned by time) executed a single input query.\n\n",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", method=\"split_by_interval_and_results_cache\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "splitting rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Intervals per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "# Query the new metric introduced in Mimir 2.10.\n(\n  sum by(request_type) (rate(cortex_frontend_query_result_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n  /\n  sum by(request_type) (rate(cortex_frontend_query_result_cache_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n)\n# Otherwise fallback to the previous general-purpose metrics.\nor\n(\n  label_replace(\n    # Query metrics before and after migration to new memcached backend.\n    sum (\n      rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n      or\n      rate(thanos_cache_memcached_hits_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n    )\n    /\n    sum (\n      rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n      or\n      rate(thanos_cache_memcached_requests_total{name=~\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n    ),\n    \"request_type\", \"query_range\", \"\", \"\")\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{request_type}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Query results cache hit ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Query results cache skipped\nThe % of queries whose results could not be cached.\nIt is tracked for each split query when the splitting by interval is enabled.\n\n",
                      "fill": 10,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_frontend_query_result_cache_skipped_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (reason) /\nignoring (reason) group_left sum(rate(cortex_frontend_query_result_cache_attempted_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "{{reason}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Query results cache skipped",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-frontend - query splitting and results cache",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Sharded queries ratio\nThe % of queries that have been successfully rewritten and executed in a shardable way.\nThis panel only takes into account the type of queries that are supported by query sharding (eg. range queries).\n\n",
                      "fill": 1,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_frontend_query_sharding_rewrites_succeeded_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) /\nsum(rate(cortex_frontend_query_sharding_rewrites_attempted_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "sharded queries ratio",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Sharded queries ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Number of sharded queries per query\nThe number of sharded queries that have been executed for a single input query. It only tracks queries that\nhave been successfully rewritten in a shardable way.\n\n",
                      "fill": 1,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_frontend_sharded_queries_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_frontend_sharded_queries_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Number of sharded queries per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-frontend - query sharding",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1 * sum(cluster_job:cortex_ingester_queried_series_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_series_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Series per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1 * sum(cluster_job:cortex_ingester_queried_samples_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_samples_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Samples per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_exemplars_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})) * 1",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1 * sum(cluster_job:cortex_ingester_queried_exemplars_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}) / sum(cluster_job:cortex_ingester_queried_exemplars_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Exemplars per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 16,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Number of store-gateways hit per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Refetches of missing blocks per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "Failure Rate": "#E24D42"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Consistency checks failed\nRate of queries that had to run with consistency checks and those checks failed. A failed consistency check means that some of at least one block which had to be queried wasn't present in any of the store-gateways.\n\n",
                      "fill": 1,
                      "id": 18,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Failure Rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Consistency checks failed",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Rejected queries\nThe proportion of all queries received by queriers that were rejected for some reason.\n\n",
                      "fill": 1,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (reason) (rate(cortex_querier_queries_rejected_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) / ignoring (reason) group_left sum(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_query(_range)?\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{reason}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Rejected queries",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Querier",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 20,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
                            "format": "time_series",
                            "legendFormat": "Max",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
                            "format": "time_series",
                            "legendFormat": "Min",
                            "legendLink": null
                         },
                         {
                            "expr": "avg(cortex_bucket_index_loaded{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Bucket indexes loaded (per querier)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_bucket_index_loads_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_bucket_index_load_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Bucket indexes load / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 22,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Bucket indexes load latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 23,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "blocks",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Blocks queried / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 24,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(data_type) (\n  # Exclude \"chunks refetched\".\n  rate(cortex_bucket_store_series_data_size_fetched_bytes_sum{component=\"store-gateway\", stage!=\"refetched\", cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{data_type}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Data fetched / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "binBps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 25,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(data_type) (\n  # Exclude \"chunks processed\" to only count \"chunks returned\", other than postings and series.\n  rate(cortex_bucket_store_series_data_size_touched_bytes_sum{component=\"store-gateway\", stage!=\"processed\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{data_type}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Data touched / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "binBps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Store-gateway",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 26,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum by(stage) (rate(cortex_bucket_store_series_request_stage_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "{{stage}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Series request average latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 27,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by(stage, le) (rate(cortex_bucket_store_series_request_stage_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])))\n",
                            "format": "time_series",
                            "legendFormat": "{{stage}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Series request 99th percentile latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Series batch preloading efficiency\nThis panel shows the % of time reduced by preloading, for Series() requests which have been\nsplit to 2+ batches. If a Series() request is served within a single batch, then preloading\nis not triggered, and thus not counted in this measurement.\n\n",
                      "fill": 1,
                      "id": 28,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "# Clamping min to 0 because if preloading not useful at all, then the actual value we get is\n# slightly negative because of the small overhead introduced by preloading.\nclamp_min(1 - (\n    sum(rate(cortex_bucket_store_series_batch_preloading_wait_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n    /\n    sum(rate(cortex_bucket_store_series_batch_preloading_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n), 0)\n",
                            "format": "time_series",
                            "legendFormat": "% of time reduced by preloading",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Series batch preloading efficiency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Blocks currently owned\nThis panel shows the number of blocks owned by each store-gateway replica.\nFor each owned block, the store-gateway keeps its index-header on disk, and\neventually loaded in memory (if index-header lazy loading is disabled, or lazy loading\nis enabled and the index-header was loaded).\n\n",
                      "fill": 0,
                      "id": 29,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "cortex_bucket_store_blocks_loaded{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Blocks currently owned",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 30,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Blocks loaded / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 31,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Blocks dropped / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 32,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "cortex_bucket_store_indexheader_lazy_load_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"} - cortex_bucket_store_indexheader_lazy_unload_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Lazy loaded index-headers",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 33,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Index-header lazy load duration",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Index-header lazy load gate latency\nTime spent waiting for a turn to load an index header. This time is not included in \"Index-header lazy load duration.\"\n\n",
                      "fill": 1,
                      "id": 34,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_stores_gate_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_stores_gate_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_bucket_stores_gate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_stores_gate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",gate=\"index_header\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Index-header lazy load gate latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 35,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_bucket_store_series_hash_cache_hits_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_bucket_store_series_hash_cache_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "hit ratio",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Series hash cache hit ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 36,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(thanos_store_index_cache_hits_total{item_type=\"ExpandedPostings\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(thanos_store_index_cache_requests_total{item_type=\"ExpandedPostings\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "hit ratio",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "ExpandedPostings cache hit ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 37,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_cache_memory_hits_total{name=\"chunks-attributes-cache\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_cache_memory_requests_total{name=\"chunks-attributes-cache\",cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "hit ratio",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Chunks attributes in-memory cache hit ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Queries",
          "uid": "b3abe8d5c040395cc36615cb4334c92d",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-reads-networking-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-reads-networking.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read).*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Summary",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-frontend.*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-frontend",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?query-scheduler.*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-scheduler",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 16,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?querier.*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Querier",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 18,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 20,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?store-gateway.*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Store-gateway",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 22,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 23,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 24,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ruler.*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ruler",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Reads networking",
          "uid": "54b2a0a4748b3bd1aefa92ce5559a1c2",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-reads-resources-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-reads-resources.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend|querier|ruler-query-frontend|ruler-querier|mimir-read\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Summary",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-frontend",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-scheduler\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-scheduler",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"querier\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Querier",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (RSS)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 16,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Rules",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 18,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ruler",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 20,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 22,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Store-gateway",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 23,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (RSS)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 24,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"store-gateway\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 25,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_written_bytes_total[$__rate_interval]\n  )\n)\n+\nignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"store-gateway\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk writes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 26,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_read_bytes_total[$__rate_interval]\n  )\n) + ignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"store-gateway\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk reads",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 27,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(store-gateway).*\"\n  }\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{persistentvolumeclaim}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk space utilization",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Reads resources",
          "uid": "cc86fd5aa9301c6528986572ad974db9",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-reads-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-reads.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "175px",
                "panels": [
                   {
                      "content": "<p>\n  This dashboard shows health metrics for the read path.\n  It is broken into sections for each service on the read path, and organized by the order in which the read request flows.\n  <br/>\n  Incoming queries travel from the gateway → query frontend → query scheduler → querier → ingester and/or store-gateway (depending on the time range of the query).\n  <br/>\n  For each service, there are 3 panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service.\n</p>\n<p>\n  The dashboard also shows metrics for the 4 optional caches that can be deployed:\n  the query results cache, the metadata cache, the chunks cache, and the index cache.\n  <br/>\n  These panels will show “no data” if the caches are not deployed.\n</p>\n<p>\n  Lastly, it also includes metrics for how the ingester and store-gateway interact with object storage.\n</p>\n",
                      "datasource": null,
                      "description": "",
                      "id": 1,
                      "mode": "markdown",
                      "span": 12,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Reads dashboard description",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "100px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Instant queries per second\nRate of instant queries per second being made to the system.\nIncludes both queries made to the <tt>/prometheus</tt> API as\nwell as queries from the ruler.\n\n",
                      "fill": 1,
                      "format": "reqps",
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  rate(\n    cortex_request_duration_seconds_count{\n      cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",\n      route=~\"(prometheus|api_prom)_api_v1_query\"\n    }[$__rate_interval]\n  )\n  or\n  rate(\n    cortex_prometheus_rule_evaluations_total{\n      cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"\n    }[$__rate_interval]\n  )\n)\n",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Instant queries / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Range queries per second\nRate of range queries per second being made to\nMimir via the <tt>/prometheus</tt> API.\n\n",
                      "fill": 1,
                      "format": "reqps",
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_query_range\"}[$__rate_interval]))",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Range queries / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### \"Label names\" queries per second\nRate of \"label names\" endpoint queries per second being made to\nMimir via the <tt>/prometheus</tt> API.\n\n",
                      "fill": 1,
                      "format": "reqps",
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_labels\"}[$__rate_interval]))",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Label names queries / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### \"Label values\" queries per second\nRate of specific \"label values\" endpoint queries per second being made to\nMimir via the <tt>/prometheus</tt> API.\n\n",
                      "fill": 1,
                      "format": "reqps",
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_label_name_values\"}[$__rate_interval]))",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Label values queries / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Series queries per second\nRate of series queries per second being made to\nMimir via the <tt>/prometheus</tt> API.\n\n",
                      "fill": 1,
                      "format": "reqps",
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\",route=~\"(prometheus|api_prom)_api_v1_series\"}[$__rate_interval]))",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Series queries / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Headlines",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "custom": {
                               "drawStyle": "line",
                               "fillOpacity": 0,
                               "lineWidth": 1,
                               "pointSize": 5,
                               "showPoints": "never",
                               "spanNulls": false,
                               "stacking": {
                                  "group": "A",
                                  "mode": "none"
                               }
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [ ]
                      },
                      "id": 9,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "displayMode": "hidden",
                            "showLegend": false
                         },
                         "tooltip": {
                            "mode": "multi",
                            "sort": "desc"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "exemplar": true,
                            "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
                            "format": "time_series",
                            "legendFormat": "",
                            "legendLink": null
                         }
                      ],
                      "title": "Per pod p99 latency",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-frontend",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Requests / sec\n<p>\n  The query scheduler is an optional service that moves\n  the internal queue from the query-frontend into a\n  separate component.\n  If this service is not deployed,\n  these panels will show \"No data.\"\n</p>\n\n",
                      "fill": 10,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Latency (Time in Queue)\n<p>\n  The query scheduler is an optional service that moves\n  the internal queue from the query-frontend into a\n  separate component.\n  If this service is not deployed,\n  these panels will show \"No data.\"\n</p>\n\n",
                      "fill": 1,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency (Time in Queue)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Latency (Time in Queue) by Queue Dimension\n<p>\n  The query scheduler is an optional service that moves\n  the internal queue from the query-frontend into a\n  separate component.\n  If this service is not deployed,\n  these panels will show \"No data.\"\n</p>\n\n",
                      "fill": 1,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "label_replace(histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
                            "format": "time_series",
                            "legendFormat": "99th Percentile: {{ additional_queue_dimensions }}",
                            "refId": "A"
                         },
                         {
                            "expr": "label_replace(histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
                            "format": "time_series",
                            "legendFormat": "50th Percentile: {{ additional_queue_dimensions }}",
                            "refId": "B"
                         },
                         {
                            "expr": "label_replace(sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (additional_queue_dimensions) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\"}[$__rate_interval])) by (additional_queue_dimensions), \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
                            "format": "time_series",
                            "legendFormat": "Average: {{ additional_queue_dimensions }}",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency (Time in Queue) by Queue Dimension",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-scheduler",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum (\n  rate(thanos_memcached_operations_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n  or ignoring(backend)\n  rate(thanos_cache_operations_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "Requests/s",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "99th Percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "50th Percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) * 1e3\n/\nsum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Cache – query results",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 16,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "custom": {
                               "drawStyle": "line",
                               "fillOpacity": 0,
                               "lineWidth": 1,
                               "pointSize": 5,
                               "showPoints": "never",
                               "spanNulls": false,
                               "stacking": {
                                  "group": "A",
                                  "mode": "none"
                               }
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [ ]
                      },
                      "id": 17,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "displayMode": "hidden",
                            "showLegend": false
                         },
                         "tooltip": {
                            "mode": "multi",
                            "sort": "desc"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "exemplar": true,
                            "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
                            "format": "time_series",
                            "legendFormat": "",
                            "legendLink": null
                         }
                      ],
                      "title": "Per pod p99 latency",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Querier",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 18,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "custom": {
                               "drawStyle": "line",
                               "fillOpacity": 0,
                               "lineWidth": 1,
                               "pointSize": 5,
                               "showPoints": "never",
                               "spanNulls": false,
                               "stacking": {
                                  "group": "A",
                                  "mode": "none"
                               }
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [ ]
                      },
                      "id": 20,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "displayMode": "hidden",
                            "showLegend": false
                         },
                         "tooltip": {
                            "mode": "multi",
                            "sort": "desc"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "exemplar": true,
                            "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval])))",
                            "format": "time_series",
                            "legendFormat": "",
                            "legendLink": null
                         }
                      ],
                      "title": "Per pod p99 latency",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 22,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "custom": {
                               "drawStyle": "line",
                               "fillOpacity": 0,
                               "lineWidth": 1,
                               "pointSize": 5,
                               "showPoints": "never",
                               "spanNulls": false,
                               "stacking": {
                                  "group": "A",
                                  "mode": "none"
                               }
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [ ]
                      },
                      "id": 23,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "displayMode": "hidden",
                            "showLegend": false
                         },
                         "tooltip": {
                            "mode": "multi",
                            "sort": "desc"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "exemplar": true,
                            "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))",
                            "format": "time_series",
                            "legendFormat": "",
                            "legendLink": null
                         }
                      ],
                      "title": "Per pod p99 latency",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Store-gateway",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Replicas\nThe maximum, and current number of querier replicas.\nPlease note that the current number of replicas can still show 1 replica even when scaled to 0.\nSince HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
                      "fill": 1,
                      "id": 24,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "/Max .+/",
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "/Current .+/",
                            "fill": 0
                         },
                         {
                            "alias": "/Min .+/",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n  # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Max {{ scaletargetref_name }}",
                            "legendLink": null
                         },
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n  # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active.\n  * on (cluster, namespace, horizontalpodautoscaler)\n    kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\", condition=\"ScalingActive\", status=\"true\"}\n  # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Current {{ scaletargetref_name }}",
                            "legendLink": null
                         },
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n  # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Min {{ scaletargetref_name }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Replicas",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Scaling metric (desired replicas)\nThis panel shows the result scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints which are applied later.\n\n",
                      "fill": 1,
                      "id": 25,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "Target per replica",
                            "yaxis": 2
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (scaler) (\n  label_replace(\n    keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\"},\n    \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n  )\n  /\n  on(cluster, namespace, scaledObject, metric) group_left\n  label_replace(label_replace(\n      kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"},\n      \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n  ), \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\")\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{ scaler }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Scaling metric (desired replicas)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": true
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
                      "fill": 1,
                      "id": 26,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n  label_replace(\n    rate(keda_scaler_errors[$__rate_interval]),\n    \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n  )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n  label_replace(\n      kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-querier\"} * 0,\n      \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n  ),\n  \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{scaler}} failures",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Autoscaler failures rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Querier - autoscaling",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 27,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 28,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\", kv_name=~\"store-gateway\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Store-gateway – key-value store for store-gateways ring",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 29,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (\n  # Backwards compatibility\n  rate(\n    thanos_memcached_operations_total{\n      component=\"store-gateway\",\n      name=\"index-cache\",\n      cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n    }[$__rate_interval]\n  )\n  or ignoring(backend)\n  rate(\n    thanos_cache_operations_total{\n      component=\"store-gateway\",\n      name=\"index-cache\",\n      cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n    }[$__rate_interval]\n  )\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 30,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"index-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "99th Percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"index-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "50th Percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_sum{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"index-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_sum{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"index-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_count{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"index-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_count{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"index-cache\"\n}\n[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency (getmulti)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Hit ratio\nEven if you do not set up memcached for the blocks index cache, you will still see data in this panel because the store-gateway by default has an\nin-memory blocks index cache.\n\n",
                      "fill": 1,
                      "id": 31,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(item_type) (\n  rate(\n    thanos_store_index_cache_hits_total{\n      component=\"store-gateway\",\n      cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n    }[$__rate_interval]\n  )\n)\n/\nsum by(item_type) (\n  rate(\n    thanos_store_index_cache_requests_total{\n      component=\"store-gateway\",\n      cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n    }[$__rate_interval]\n  )\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{item_type}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Hit ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Memcached – block index cache (store-gateway accesses)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 32,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (\n  # Backwards compatibility\n  rate(thanos_memcached_operations_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"chunks-cache\"\n  }[$__rate_interval])\n  or ignoring(backend)\n  rate(thanos_cache_operations_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"chunks-cache\"\n  }[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 33,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"chunks-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "99th Percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"chunks-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "50th Percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_sum{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"chunks-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_sum{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"chunks-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_count{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"chunks-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_count{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"chunks-cache\"\n}\n[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency (getmulti)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 34,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  # Backwards compatibility\n  rate(thanos_cache_memcached_hits_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"chunks-cache\"\n  }[$__rate_interval])\n  or\n  rate(thanos_cache_hits_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"chunks-cache\"\n  }[$__rate_interval])\n)\n/\nsum(\n  # Backwards compatibility\n  rate(thanos_cache_memcached_requests_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"chunks-cache\"\n  }[$__rate_interval])\n  or\n  rate(thanos_cache_requests_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"chunks-cache\"\n  }[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "items",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Hit ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Memcached – chunks cache (store-gateway accesses)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 35,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (\n  # Backwards compatibility\n  rate(thanos_memcached_operations_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n  or ignoring(backend)\n  rate(thanos_cache_operations_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 36,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "99th Percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "50th Percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_sum{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_sum{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_count{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_count{\n  cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n  operation=\"getmulti\",\n  component=\"store-gateway\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency (getmulti)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 37,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  # Backwards compatibility\n  rate(thanos_cache_memcached_hits_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n  or\n  rate(thanos_cache_hits_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n)\n/\nsum(\n  # Backwards compatibility\n  rate(thanos_cache_memcached_requests_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n  or\n  rate(thanos_cache_requests_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n    component=\"store-gateway\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "items",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Hit ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Memcached – metadata cache (store-gateway accesses)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 38,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (\n  # Backwards compatibility\n  rate(thanos_memcached_operations_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n    component=\"querier\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n  or ignoring(backend)\n  rate(thanos_cache_operations_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n    component=\"querier\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 39,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n  operation=\"getmulti\",\n  component=\"querier\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n  operation=\"getmulti\",\n  component=\"querier\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "99th Percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n  operation=\"getmulti\",\n  component=\"querier\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_bucket{\n  cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n  operation=\"getmulti\",\n  component=\"querier\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "50th Percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "sum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_sum{\n  cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n  operation=\"getmulti\",\n  component=\"querier\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_sum{\n  cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n  operation=\"getmulti\",\n  component=\"querier\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n  # Backwards compatibility\n  rate(thanos_memcached_operation_duration_seconds_count{\n  cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n  operation=\"getmulti\",\n  component=\"querier\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n  or\n  rate(thanos_cache_operation_duration_seconds_count{\n  cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n  operation=\"getmulti\",\n  component=\"querier\",\n  name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency (getmulti)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 40,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  # Backwards compatibility\n  rate(thanos_cache_memcached_hits_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n    component=\"querier\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n  or\n  rate(thanos_cache_hits_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n    component=\"querier\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n)\n/\nsum(\n  # Backwards compatibility\n  rate(thanos_cache_memcached_requests_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n    component=\"querier\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n  or\n  rate(thanos_cache_requests_total{\n    cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n    component=\"querier\",\n    name=\"metadata-cache\"\n  }[$__rate_interval])\n)\n",
                            "format": "time_series",
                            "legendFormat": "items",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Hit ratio",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Memcached – metadata cache (querier accesses)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 41,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Operations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "max": 1,
                            "min": 0,
                            "noValue": "0",
                            "unit": "percentunit"
                         }
                      },
                      "id": 42,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 3,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\"}[$__rate_interval])) >= 0",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Error rate",
                      "type": "timeseries"
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 43,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Attributes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 44,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Exists",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Blocks object store (store-gateway accesses)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 45,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Get",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 46,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: GetRange",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 47,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Upload",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 48,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Delete",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 49,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Operations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "max": 1,
                            "min": 0,
                            "noValue": "0",
                            "unit": "percentunit"
                         }
                      },
                      "id": 50,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 3,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\"}[$__rate_interval])) >= 0",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Error rate",
                      "type": "timeseries"
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 51,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"attributes\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Attributes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 52,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"exists\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Exists",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Blocks object store (querier accesses)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 53,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Get",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 54,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"get_range\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: GetRange",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 55,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"upload\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Upload",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 56,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"querier\",operation=\"delete\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Delete",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Reads",
          "uid": "e327503188913dc38ad571c647eef643",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-remote-ruler-reads-resources-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-remote-ruler-reads-resources.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-frontend (dedicated to ruler)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-scheduler (dedicated to ruler)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Querier (dedicated to ruler)",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Remote ruler reads resources",
          "uid": "1940f6ef765a506a171faa2056c956c3",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-remote-ruler-reads-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-remote-ruler-reads.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "175px",
                "panels": [
                   {
                      "content": "<p>\n  This dashboard shows health metrics for the ruler read path when remote operational mode is enabled.\n  It is broken into sections for each service on the ruler read path, and organized by the order in which the read request flows.\n  <br/>\n  For each service, there are three panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service.\n</p>\n",
                      "datasource": null,
                      "description": "",
                      "id": 1,
                      "mode": "markdown",
                      "span": 12,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Remote ruler reads dashboard description",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "100px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Evaluations per second\nRate of rule expressions evaluated per second.\n\n",
                      "fill": 1,
                      "format": "reqps",
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  rate(\n    cortex_request_duration_seconds_count{\n      cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\",\n      route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"\n    }[$__rate_interval]\n  )\n)\n",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Evaluations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Headlines",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "custom": {
                               "drawStyle": "line",
                               "fillOpacity": 0,
                               "lineWidth": 1,
                               "pointSize": 5,
                               "showPoints": "never",
                               "spanNulls": false,
                               "stacking": {
                                  "group": "A",
                                  "mode": "none"
                               }
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [ ]
                      },
                      "id": 5,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "displayMode": "hidden",
                            "showLegend": false
                         },
                         "tooltip": {
                            "mode": "multi",
                            "sort": "desc"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "exemplar": true,
                            "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle|.*api_v1_query\"}[$__rate_interval])))",
                            "format": "time_series",
                            "legendFormat": "",
                            "legendLink": null
                         }
                      ],
                      "title": "Per pod p99 latency",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-frontend (dedicated to ruler)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Requests / sec\n<p>\n  The query scheduler is an optional service that moves\n  the internal queue from the query-frontend into a\n  separate component.\n  If this service is not deployed,\n  these panels will show \"No data.\"\n</p>\n\n",
                      "fill": 10,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Latency (Time in Queue)\n<p>\n  The query scheduler is an optional service that moves\n  the internal queue from the query-frontend into a\n  separate component.\n  If this service is not deployed,\n  these panels will show \"No data.\"\n</p>\n\n",
                      "fill": 1,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency (Time in Queue)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Latency (Time in Queue) by Queue Dimension\n<p>\n  The query scheduler is an optional service that moves\n  the internal queue from the query-frontend into a\n  separate component.\n  If this service is not deployed,\n  these panels will show \"No data.\"\n</p>\n\n",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "label_replace(histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
                            "format": "time_series",
                            "legendFormat": "99th Percentile: {{ additional_queue_dimensions }}",
                            "refId": "A"
                         },
                         {
                            "expr": "label_replace(histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (le, additional_queue_dimensions)) * 1e3, \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
                            "format": "time_series",
                            "legendFormat": "50th Percentile: {{ additional_queue_dimensions }}",
                            "refId": "B"
                         },
                         {
                            "expr": "label_replace(sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (additional_queue_dimensions) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\"}[$__rate_interval])) by (additional_queue_dimensions), \"additional_queue_dimensions\", \"none\", \"additional_queue_dimensions\", \"^$\")\n",
                            "format": "time_series",
                            "legendFormat": "Average: {{ additional_queue_dimensions }}",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency (Time in Queue) by Queue Dimension",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Query-scheduler (dedicated to ruler)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "custom": {
                               "drawStyle": "line",
                               "fillOpacity": 0,
                               "lineWidth": 1,
                               "pointSize": 5,
                               "showPoints": "never",
                               "spanNulls": false,
                               "stacking": {
                                  "group": "A",
                                  "mode": "none"
                               }
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [ ]
                      },
                      "id": 11,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "displayMode": "hidden",
                            "showLegend": false
                         },
                         "tooltip": {
                            "mode": "multi",
                            "sort": "desc"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "exemplar": true,
                            "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))",
                            "format": "time_series",
                            "legendFormat": "",
                            "legendLink": null
                         }
                      ],
                      "title": "Per pod p99 latency",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Querier (dedicated to ruler)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Replicas\nThe maximum and current number of ruler-querier replicas.\nNote: The current number of replicas can still show 1 replica even when scaled to 0.\nBecause HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
                      "fill": 1,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "/Max .+/",
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "/Current .+/",
                            "fill": 0
                         },
                         {
                            "alias": "/Min .+/",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n  # Add the scaletargetref_name label for readability\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Max {{ scaletargetref_name }}",
                            "legendLink": null
                         },
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n  # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active\n  * on (cluster, namespace, horizontalpodautoscaler)\n    kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\", condition=\"ScalingActive\", status=\"true\"}\n  # Add the scaletargetref_name label for readability\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Current {{ scaletargetref_name }}",
                            "legendLink": null
                         },
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n  # Add the scaletargetref_name label for readability\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Min {{ scaletargetref_name }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Replicas",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Scaling metric (CPU): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (scaler) (\n  label_replace(\n    keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n    \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n  )\n  /\n  on(cluster, namespace, scaledObject, metric) group_left label_replace(\n    label_replace(\n      kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n      \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n    ),\n    \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n  )\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{ scaler }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Scaling metric (CPU): Desired replicas",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Scaling metric (memory): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (scaler) (\n  label_replace(\n    keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n    \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n  )\n  /\n  on(cluster, namespace, scaledObject, metric) group_left label_replace(\n    label_replace(\n      kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"},\n      \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n    ),\n    \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n  )\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{ scaler }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Scaling metric (memory): Desired replicas",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
                      "fill": 1,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n  label_replace(\n    rate(keda_scaler_errors[$__rate_interval]),\n    \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n  )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n  label_replace(\n      kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-ruler-querier\"} * 0,\n      \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n  ),\n  \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{scaler}} failures",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Autoscaler failures rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ruler-Querier - autoscaling",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Remote ruler reads",
          "uid": "f103238f7f5ab2f1345ce650cbfbfe2f",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-rollout-progress-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-rollout-progress.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "panels": [
             {
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "custom": {
                         "axisCenteredZero": false,
                         "axisColorMode": "text",
                         "axisLabel": "",
                         "axisPlacement": "auto",
                         "fillOpacity": 80,
                         "gradientMode": "none",
                         "hideFrom": {
                            "legend": false,
                            "tooltip": false,
                            "viz": false
                         },
                         "lineWidth": 1,
                         "scaleDistribution": {
                            "type": "linear"
                         },
                         "thresholdsStyle": {
                            "mode": "off"
                         }
                      },
                      "mappings": [ ],
                      "max": 1,
                      "min": 0,
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [ ]
                      },
                      "unit": "percentunit"
                   },
                   "overrides": [
                      {
                         "matcher": {
                            "id": "byName",
                            "options": "Ready"
                         },
                         "properties": [
                            {
                               "id": "color",
                               "value": {
                                  "fixedColor": "green",
                                  "mode": "fixed"
                               }
                            }
                         ]
                      },
                      {
                         "matcher": {
                            "id": "byName",
                            "options": "Updated"
                         },
                         "properties": [
                            {
                               "id": "color",
                               "value": {
                                  "fixedColor": "blue",
                                  "mode": "fixed"
                               }
                            }
                         ]
                      }
                   ]
                },
                "gridPos": {
                   "h": 13,
                   "w": 10,
                   "x": 0,
                   "y": 0
                },
                "id": 1,
                "links": [ ],
                "options": {
                   "barRadius": 0,
                   "barWidth": 0.96999999999999997,
                   "fullHighlight": false,
                   "groupWidth": 0.69999999999999996,
                   "legend": {
                      "calcs": [ ],
                      "displayMode": "list",
                      "placement": "bottom",
                      "showLegend": true
                   },
                   "orientation": "horizontal",
                   "showValue": "auto",
                   "stacking": "none",
                   "tooltip": {
                      "mode": "multi",
                      "sort": "none"
                   },
                   "xField": "Workload",
                   "xTickLabelRotation": 0,
                   "xTickLabelSpacing": 0
                },
                "targets": [
                   {
                      "expr": "(\n  sum by(cortex_service) (\n    label_replace(\n      label_replace(\n        label_replace(\n          {__name__=~\"kube_(deployment|statefulset)_status_replicas_updated\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n          \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n        ),\n        \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n      ),\n      # Strip the -zone-X suffix, if there is one\n      \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n  /\n  sum by(cortex_service) (\n    label_replace(\n      label_replace(\n        label_replace(\n          {__name__=~\"kube_(deployment|statefulset)_status_replicas\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n          \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n        ),\n        \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n      ),\n      # Strip the -zone-X suffix, if there is one\n      \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n) and (\n  sum by(cortex_service) (\n    label_replace(\n      label_replace(\n        label_replace(\n          {__name__=~\"kube_(deployment|statefulset)_status_replicas\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n          \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n        ),\n        \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n      ),\n      # Strip the -zone-X suffix, if there is one\n      \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n  > 0\n)\n",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": null,
                      "legendFormat": "__auto",
                      "legendLink": null,
                      "step": null
                   },
                   {
                      "expr": "(\n  sum by(cortex_service) (\n    label_replace(\n      label_replace(\n        label_replace(\n          {__name__=~\"kube_(deployment|statefulset)_status_replicas_ready\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n          \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n        ),\n        \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n      ),\n      # Strip the -zone-X suffix, if there is one\n      \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n  /\n  sum by(cortex_service) (\n    label_replace(\n      label_replace(\n        label_replace(\n          {__name__=~\"kube_(deployment|statefulset)_status_replicas\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n          \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n        ),\n        \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n      ),\n      # Strip the -zone-X suffix, if there is one\n      \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n) and (\n  sum by(cortex_service) (\n    label_replace(\n      label_replace(\n        label_replace(\n          {__name__=~\"kube_(deployment|statefulset)_status_replicas\", cluster=~\"$cluster\", namespace=~\"$namespace\"},\n          \"cortex_service\", \"$1\", \"deployment\", \"(.+)\"\n        ),\n        \"cortex_service\", \"$1\", \"statefulset\", \"(.+)\"\n      ),\n      # Strip the -zone-X suffix, if there is one\n      \"cortex_service\", \"$1\", \"cortex_service\", \"(.*?)(?:-zone-[a-z])?\"\n    )\n  )\n  > 0\n)\n",
                      "format": "table",
                      "instant": true,
                      "intervalFactor": null,
                      "legendFormat": "__auto",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "title": "Rollout progress",
                "transformations": [
                   {
                      "id": "joinByField",
                      "options": {
                         "byField": "cortex_service",
                         "mode": "outer"
                      }
                   },
                   {
                      "id": "organize",
                      "options": {
                         "excludeByName": {
                            "Time 1": true,
                            "Time 2": true
                         },
                         "renameByName": {
                            "Value #A": "Updated",
                            "Value #B": "Ready",
                            "cortex_service": "Workload"
                         }
                      }
                   },
                   {
                      "id": "sortBy",
                      "options": {
                         "sort": [
                            {
                               "field": "Workload"
                            }
                         ]
                      }
                   }
                ],
                "type": "barchart"
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 1,
                      "noValue": "",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            }
                         ]
                      },
                      "unit": "percentunit"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 4,
                   "w": 2,
                   "x": 10,
                   "y": 0
                },
                "id": 2,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
                      "format": null,
                      "instant": false,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Writes - 2xx",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 1,
                      "noValue": "",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            },
                            {
                               "color": "orange",
                               "value": 0.20000000000000001
                            },
                            {
                               "color": "red",
                               "value": 0.5
                            }
                         ]
                      },
                      "unit": "percentunit"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 4,
                   "w": 2,
                   "x": 12,
                   "y": 0
                },
                "id": 3,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
                      "format": null,
                      "instant": false,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Writes - 4xx",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 1,
                      "noValue": "",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            },
                            {
                               "color": "red",
                               "value": 0.01
                            }
                         ]
                      },
                      "unit": "percentunit"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 4,
                   "w": 2,
                   "x": 14,
                   "y": 0
                },
                "id": 4,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]))\n",
                      "format": null,
                      "instant": false,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Writes - 5xx",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 1,
                      "noValue": "",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            },
                            {
                               "color": "orange",
                               "value": 0.20000000000000001
                            },
                            {
                               "color": "red",
                               "value": 0.5
                            }
                         ]
                      },
                      "unit": "s"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 4,
                   "w": 8,
                   "x": 16,
                   "y": 0
                },
                "id": 5,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}))\n",
                      "format": null,
                      "instant": false,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Writes 99th latency",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 1,
                      "noValue": "",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            }
                         ]
                      },
                      "unit": "percentunit"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 4,
                   "w": 2,
                   "x": 10,
                   "y": 4
                },
                "id": 6,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
                      "format": null,
                      "instant": false,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Reads - 2xx",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 1,
                      "noValue": "",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            },
                            {
                               "color": "orange",
                               "value": 0.01
                            },
                            {
                               "color": "red",
                               "value": 0.050000000000000003
                            }
                         ]
                      },
                      "unit": "percentunit"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 4,
                   "w": 2,
                   "x": 12,
                   "y": 4
                },
                "id": 7,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
                      "format": null,
                      "instant": false,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Reads - 4xx",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 1,
                      "noValue": "",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            },
                            {
                               "color": "red",
                               "value": 0.01
                            }
                         ]
                      },
                      "unit": "percentunit"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 4,
                   "w": 2,
                   "x": 14,
                   "y": 4
                },
                "id": 8,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n",
                      "format": null,
                      "instant": false,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Reads - 5xx",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 1,
                      "noValue": "",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            },
                            {
                               "color": "orange",
                               "value": 1
                            },
                            {
                               "color": "red",
                               "value": 2.5
                            }
                         ]
                      },
                      "unit": "s"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 4,
                   "w": 8,
                   "x": 16,
                   "y": 4
                },
                "id": 9,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))\n",
                      "format": null,
                      "instant": false,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Reads 99th latency",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "aliasColors": { },
                "bars": false,
                "dashLength": 10,
                "dashes": false,
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "color": {
                         "mode": "thresholds"
                      },
                      "decimals": 0,
                      "noValue": "All healthy",
                      "thresholds": {
                         "mode": "absolute",
                         "steps": [
                            {
                               "color": "green",
                               "value": null
                            },
                            {
                               "color": "orange",
                               "value": 1
                            },
                            {
                               "color": "red",
                               "value": 2
                            }
                         ]
                      },
                      "unit": "short"
                   },
                   "overrides": [ ]
                },
                "fill": 1,
                "gridPos": {
                   "h": 3,
                   "w": 10,
                   "x": 0,
                   "y": 13
                },
                "id": 10,
                "legend": {
                   "avg": false,
                   "current": false,
                   "max": false,
                   "min": false,
                   "show": true,
                   "total": false,
                   "values": false
                },
                "lines": true,
                "linewidth": 1,
                "links": [ ],
                "nullPointMode": "null as zero",
                "options": {
                   "text": {
                      "titleSize": 14,
                      "valueSize": 14
                   },
                   "textMode": "value_and_name"
                },
                "percentage": false,
                "pointradius": 5,
                "points": false,
                "renderer": "flot",
                "seriesOverrides": [ ],
                "spaceLength": 10,
                "span": 6,
                "stack": false,
                "steppedLine": false,
                "targets": [
                   {
                      "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n> 0\n",
                      "format": null,
                      "instant": true,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "{{deployment}}",
                      "legendLink": null,
                      "step": null
                   },
                   {
                      "expr": "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\"}\n> 0\n",
                      "format": null,
                      "instant": true,
                      "interval": "",
                      "intervalFactor": null,
                      "legendFormat": "{{statefulset}}",
                      "legendLink": null,
                      "step": null
                   }
                ],
                "thresholds": [ ],
                "timeFrom": null,
                "timeShift": null,
                "title": "Unhealthy pods",
                "tooltip": {
                   "shared": false,
                   "sort": 0,
                   "value_type": "individual"
                },
                "type": "stat",
                "xaxis": {
                   "buckets": null,
                   "mode": "time",
                   "name": null,
                   "show": true,
                   "values": [ ]
                },
                "yaxes": [
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": 0,
                      "show": true
                   },
                   {
                      "format": "short",
                      "label": null,
                      "logBase": 1,
                      "max": null,
                      "min": null,
                      "show": false
                   }
                ]
             },
             {
                "datasource": "$datasource",
                "fieldConfig": {
                   "overrides": [
                      {
                         "matcher": {
                            "id": "byRegexp",
                            "options": "r.*"
                         },
                         "properties": [
                            {
                               "id": "custom.align",
                               "value": "center"
                            }
                         ]
                      }
                   ]
                },
                "gridPos": {
                   "h": 8,
                   "w": 6,
                   "x": 10,
                   "y": 8
                },
                "id": 11,
                "targets": [
                   {
                      "expr": "count by(container, version) (\n  label_replace(\n    kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n    \"version\", \"$1\", \"image\", \".*:(.*)\"\n  )\n)\n",
                      "instant": true,
                      "legendFormat": "",
                      "refId": "A"
                   }
                ],
                "title": "Pods count per version",
                "transformations": [
                   {
                      "id": "labelsToFields",
                      "options": {
                         "valueLabel": "version"
                      }
                   },
                   {
                      "id": "organize",
                      "options": {
                         "excludeByName": {
                            "Time": true
                         },
                         "indexByName": {
                            "Time": 0,
                            "container": 1
                         }
                      }
                   },
                   {
                      "id": "sortBy",
                      "options": {
                         "fields": { },
                         "sort": [
                            {
                               "field": "container"
                            }
                         ]
                      }
                   }
                ],
                "type": "table"
             },
             {
                "datasource": "$datasource",
                "fieldConfig": {
                   "defaults": {
                      "custom": {
                         "fillOpacity": 10
                      },
                      "unit": "percentunit"
                   }
                },
                "gridPos": {
                   "h": 8,
                   "w": 8,
                   "x": 16,
                   "y": 8
                },
                "id": 12,
                "links": [ ],
                "options": {
                   "legend": {
                      "showLegend": true
                   },
                   "tooltip": {
                      "mode": "single",
                      "sort": "none"
                   }
                },
                "targets": [
                   {
                      "expr": "1 - (\n  avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"} offset 24h))[1h:])\n  /\n  avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"api_(v1|prom)_push|otlp_v1_metrics\"}))[1h:])\n)\n",
                      "format": "time_series",
                      "legendFormat": "writes",
                      "legendLink": null
                   },
                   {
                      "expr": "1 - (\n  avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"} offset 24h))[1h:])\n  /\n  avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))[1h:])\n)\n",
                      "format": "time_series",
                      "legendFormat": "reads",
                      "legendLink": null
                   }
                ],
                "title": "Latency vs 24h ago",
                "type": "timeseries"
             }
          ],
          "refresh": "10s",
          "rows": null,
          "schemaVersion": 27,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Rollout progress",
          "uid": "7f0b5567d543a1698e695b530eb7f5de",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-ruler-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-ruler.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "100px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "format": "short",
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cortex_ruler_managers_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Active configurations",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "format": "short",
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"})",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Total rules",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Read from ingesters - QPS\nNote: Even while operating in Remote ruler mode you will still see values for this panel.\n\nThis is because the metrics are inclusive of intermediate services and are showing the requests that ultimately reach the ingesters.\n\nFor a more detailed view of the read path when using remote ruler mode, see the Remote ruler reads dashboard.\n\n",
                      "fill": 1,
                      "format": "reqps",
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Read from ingesters - QPS",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "format": "reqps",
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Write to ingesters - QPS",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Headlines",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "success",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "missed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Evaluations per second",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n  /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "average",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Rule evaluations global",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "QPS",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Writes (ingesters)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "QPS",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*|ruler-querier.*))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Reads (ingesters)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", kv_name=~\"ruler\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ruler - key-value store for rulers ring",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Number of store-gateways hit per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Refetches of missing blocks per query",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "Failures / sec": "#E24D42"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Consistency checks failed\nRate of queries that had to run with consistency checks and those checks failed. A failed consistency check means that some of at least one block which had to be queried wasn't present in any of the store-gateways.\n\n",
                      "fill": 1,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Failures / sec",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Consistency checks failed",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": 1,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ruler - blocks storage",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "noValue": 0,
                            "unit": "short"
                         }
                      },
                      "id": 16,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n  /\nsum by(user) (rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]) > 0)\n> 0\n",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "title": "Delivery errors",
                      "type": "timeseries"
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "noValue": 0,
                            "unit": "percentunit"
                         }
                      },
                      "fill": 1,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n  /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Queue length",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "noValue": 0,
                            "unit": "short"
                         }
                      },
                      "id": 18,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "expr": "sum by (user) (increase(cortex_prometheus_notifications_dropped_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0\n",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "title": "Dropped",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Notifications",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Missed iterations",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 20,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n  /\nrate(cortex_prometheus_rule_group_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])\n",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) > 0",
                            "format": "time_series",
                            "legendFormat": "{{ rule_group }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Failures",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Group evaluations",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 22,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n  /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))\n",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Rule evaluation per user",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 23,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Operations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "max": 1,
                            "min": 0,
                            "noValue": "0",
                            "unit": "percentunit"
                         }
                      },
                      "id": 24,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 3,
                      "targets": [
                         {
                            "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\"}[$__rate_interval])) >= 0",
                            "format": "time_series",
                            "legendFormat": "{{operation}}",
                            "legendLink": null
                         }
                      ],
                      "title": "Error rate",
                      "type": "timeseries"
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 25,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Attributes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 26,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Exists",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ruler configuration object store (ruler accesses)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 27,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Get",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 28,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: GetRange",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 29,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Upload",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 30,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency of op: Delete",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Ruler",
          "uid": "631e15d5d85afb2ca8e35d62984eeaa0",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-scaling-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-scaling.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "200px",
                "panels": [
                   {
                      "id": 1,
                      "options": {
                         "content": "This dashboard identifies scaling-related issues by suggesting services that you might want to scale up.\nThe table that follows contains a suggested number of replicas and the reason why.\nIf the system is failing and depending on the reason, try scaling up to the specified number.\nThe specified numbers are intended as helpful guidelines when things go wrong, rather than prescriptive guidelines.\n\nReasons:\n- **sample_rate**: There are not enough replicas to handle the\n  sample rate.  Applies to distributor and ingesters.\n- **active_series**: There are not enough replicas\n  to handle the number of active series.  Applies to ingesters.\n- **cpu_usage**: There are not enough replicas\n  based on the CPU usage of the jobs vs the resource requests.\n  Applies to all jobs.\n- **memory_usage**: There are not enough replicas based on the memory\n  usage vs the resource requests.  Applies to all jobs.\n- **active_series_limits**: There are not enough replicas to hold 60% of the\n  sum of all the per tenant series limits.\n- **sample_rate_limits**: There are not enough replicas to handle 60% of the\n  sum of all the per tenant rate limits.\n",
                         "mode": "markdown"
                      },
                      "span": 12,
                      "title": "",
                      "type": "text"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Service scaling",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "400px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 0,
                         "desc": false
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "Required Replicas",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 0,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "Cluster",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "__name__",
                            "thresholds": [ ],
                            "type": "hidden",
                            "unit": "short"
                         },
                         {
                            "alias": "Cluster",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "cluster",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "Service",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "deployment",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "Namespace",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "namespace",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "Reason",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "reason",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "sort_desc(\n  cluster_namespace_deployment_reason:required_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n    > ignoring(reason) group_left\n  cluster_namespace_deployment:actual_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\n",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Workload-based scaling",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Scaling",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Scaling",
          "uid": "64bbad83507b7289b514725658e10352",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-slow-queries-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-slow-queries.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p99",
                            "legendLink": null
                         },
                         {
                            "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p50",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Response time",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p99",
                            "legendLink": null
                         },
                         {
                            "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p50",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Fetched series",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p99",
                            "legendLink": null
                         },
                         {
                            "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p50",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Fetched chunks",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p99",
                            "legendLink": null
                         },
                         {
                            "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p50",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Response size",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p99",
                            "legendLink": null
                         },
                         {
                            "expr": "quantile_over_time(0.5, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by ()",
                            "format": "time_series",
                            "legendFormat": "p50",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Time span",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Accross tenants",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by (user))",
                            "format": "time_series",
                            "legendFormat": "{{user}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 response time",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by (user))",
                            "format": "time_series",
                            "legendFormat": "{{user}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 fetched series",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by (user))",
                            "format": "time_series",
                            "legendFormat": "{{user}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 fetched chunks",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by (user))",
                            "format": "time_series",
                            "legendFormat": "{{user}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 response size",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by (user))",
                            "format": "time_series",
                            "legendFormat": "{{user}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 time span",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Top 10 tenants",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(response_time) [$__auto]) by (user_agent))",
                            "format": "time_series",
                            "legendFormat": "{{user_agent}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 response time",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_series_count[$__auto]) by (user_agent))",
                            "format": "time_series",
                            "legendFormat": "{{user_agent}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 fetched series",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap fetched_chunk_bytes[$__auto]) by (user_agent))",
                            "format": "time_series",
                            "legendFormat": "{{user_agent}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 fetched chunks",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap response_size_bytes[$__auto]) by (user_agent))",
                            "format": "time_series",
                            "legendFormat": "{{user_agent}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 response size",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "${loki_datasource}",
                      "fill": 1,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "topk(10, quantile_over_time(0.99, {cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | unwrap duration_seconds(length) [$__auto]) by (user_agent))",
                            "format": "time_series",
                            "legendFormat": "{{user_agent}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "P99 time span",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Top 10 User-Agents",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "datasource": "${loki_datasource}",
                      "fieldConfig": {
                         "overrides": [
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "fetched_chunk_bytes"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "bytes"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "fetched_index_bytes"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "bytes"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "response_size_bytes"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "bytes"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "results_cache_hit_bytes"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "bytes"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "results_cache_miss_bytes"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "bytes"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "estimated_series_count"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "short"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "fetched_chunks_count"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "short"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "fetched_series_count"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "short"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "Time span"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "s"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "Duration"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "s"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "Step"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "s"
                                  }
                               ]
                            },
                            {
                               "matcher": {
                                  "id": "byName",
                                  "options": "queue_time_seconds"
                               },
                               "properties": [
                                  {
                                     "id": "unit",
                                     "value": "s"
                                  }
                               ]
                            }
                         ]
                      },
                      "height": "500px",
                      "id": 16,
                      "span": 12,
                      "targets": [
                         {
                            "expr": "{cluster=~\"$cluster\",namespace=~\"$namespace\",name=~\"query-frontend.*\"} |= \"query stats\" != \"/api/v1/read\" | logfmt | user=~\"${tenant_id}\" | user_agent=~\"${user_agent}\" | response_time > ${min_duration} | label_format response_time_seconds=\"{{ duration .response_time }}\",param_step_seconds=\"{{ div .param_step 1000 }}\",length_seconds=\"{{ duration .length }}\"",
                            "instant": false,
                            "legendFormat": "",
                            "range": true,
                            "refId": "A"
                         }
                      ],
                      "title": "Slow queries",
                      "transformations": [
                         {
                            "id": "extractFields",
                            "options": {
                               "source": "labels"
                            }
                         },
                         {
                            "id": "organize",
                            "options": {
                               "excludeByName": {
                                  "Line": true,
                                  "Time": true,
                                  "caller": true,
                                  "cluster": true,
                                  "component": true,
                                  "container": true,
                                  "gossip_ring_member": true,
                                  "host": true,
                                  "id": true,
                                  "job": true,
                                  "labels": true,
                                  "length": true,
                                  "level": true,
                                  "line": true,
                                  "method": true,
                                  "msg": true,
                                  "name": true,
                                  "namespace": true,
                                  "param_step": true,
                                  "path": true,
                                  "pod": true,
                                  "pod_template_hash": true,
                                  "query_wall_time_seconds": true,
                                  "response_time": true,
                                  "stream": true,
                                  "traceID": true,
                                  "tsNs": true
                               },
                               "indexByName": {
                                  "err": 10,
                                  "length_seconds": 3,
                                  "param_end": 5,
                                  "param_query": 8,
                                  "param_start": 4,
                                  "param_step_seconds": 7,
                                  "param_time": 6,
                                  "response_time_seconds": 9,
                                  "status": 1,
                                  "ts": 0,
                                  "user": 2
                               },
                               "renameByName": {
                                  "err": "Error",
                                  "length_seconds": "Time span",
                                  "param_end": "End",
                                  "param_query": "Query",
                                  "param_start": "Start",
                                  "param_step_seconds": "Step",
                                  "param_time": "Time (instant query)",
                                  "response_time_seconds": "Duration",
                                  "ts": "Completion date",
                                  "user": "Tenant ID"
                               }
                            }
                         },
                         {
                            "id": "convertFieldType",
                            "options": {
                               "conversions": [
                                  {
                                     "destinationType": "number",
                                     "targetField": "estimated_series_count"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "fetched_chunk_bytes"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "fetched_chunks_count"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "fetched_index_bytes"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "fetched_series_count"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "queue_time_seconds"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "response_size_bytes"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "results_cache_hit_bytes"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "results_cache_miss_bytes"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "sharded_queries"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "split_queries"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "Time span"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "Duration"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "Step"
                                  },
                                  {
                                     "destinationType": "number",
                                     "targetField": "queue_time_seconds"
                                  }
                               ]
                            }
                         }
                      ],
                      "type": "table"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "hide": 0,
                   "includeAll": false,
                   "label": "Loki data source",
                   "multi": false,
                   "name": "loki_datasource",
                   "query": "loki",
                   "type": "datasource"
                },
                {
                   "current": {
                      "selected": true,
                      "text": "5s",
                      "value": "5s"
                   },
                   "hide": 0,
                   "label": "Min duration",
                   "name": "min_duration",
                   "options": [
                      {
                         "selected": true,
                         "text": "5s",
                         "value": "5s"
                      }
                   ],
                   "query": "5s",
                   "type": "textbox"
                },
                {
                   "current": {
                      "selected": true,
                      "text": ".*",
                      "value": ".*"
                   },
                   "hide": 0,
                   "label": "Tenant ID",
                   "name": "tenant_id",
                   "options": [
                      {
                         "selected": true,
                         "text": ".*",
                         "value": ".*"
                      }
                   ],
                   "query": ".*",
                   "type": "textbox"
                },
                {
                   "current": {
                      "selected": true,
                      "text": ".*",
                      "value": ".*"
                   },
                   "hide": 0,
                   "label": "User-Agent HTTP Header",
                   "name": "user_agent",
                   "options": [
                      {
                         "selected": true,
                         "text": ".*",
                         "value": ".*"
                      }
                   ],
                   "query": ".*",
                   "type": "textbox"
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Slow queries",
          "uid": "6089e1ce1e678788f46312a0a1e647e6",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-tenants-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-tenants.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "25px",
                "panels": [
                   {
                      "content": "<p>\n  This dashboard shows various metrics detailed by tenant (user) selected above.\n</p>\n",
                      "datasource": null,
                      "description": "",
                      "id": 1,
                      "mode": "markdown",
                      "span": 12,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Tenants dashboard description",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### All series\nNumber of active and in-memory series per user, and active series matching custom trackers (in parenthesis).\nNote that these counts include all series regardless of the type of data (counter, gauge, native histogram, etc.).\nNote that active series matching custom trackers are included in the total active series count.\n\n",
                      "fill": 1,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "limit",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  (\n    cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n    - cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n  )\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
                            "format": "time_series",
                            "legendFormat": "in-memory",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"max_global_series_per_user\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"max_global_series_per_user\"})\n",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(\n  cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
                            "format": "time_series",
                            "legendFormat": "active",
                            "legendLink": null
                         },
                         {
                            "expr": "sum by (name) (\n  cortex_ingester_active_series_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
                            "format": "time_series",
                            "legendFormat": "active ({{ name }})",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "All series",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Native histogram series\nNumber of active native histogram series per user, and active native histogram series matching custom trackers (in parenthesis).\nNote that active series matching custom trackers are included in the total active series count.\n\n",
                      "fill": 1,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "limit",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  cortex_ingester_active_native_histogram_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
                            "format": "time_series",
                            "legendFormat": "active",
                            "legendLink": null
                         },
                         {
                            "expr": "sum by (name) (\n  cortex_ingester_active_native_histogram_series_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
                            "format": "time_series",
                            "legendFormat": "active ({{ name }})",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Native histogram series",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Total number of buckets used by native histogram series\nTotal number of buckets in active native histogram series per user, and total active native histogram buckets matching custom trackers (in parenthesis).\n\n",
                      "fill": 1,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "limit",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  cortex_ingester_active_native_histogram_buckets{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
                            "format": "time_series",
                            "legendFormat": "buckets",
                            "legendLink": null
                         },
                         {
                            "expr": "sum by (name) (\n  cortex_ingester_active_native_histogram_buckets_custom_tracker{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n) > 0\n",
                            "format": "time_series",
                            "legendFormat": "buckets ({{ name }})",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Total number of buckets used by native histogram series",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Active series and native histograms",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Series with exemplars\nNumber of series with exemplars currently in storage.\n\n",
                      "fill": 1,
                      "id": 5,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
                            "format": "time_series",
                            "legendFormat": "series",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Series with exemplars",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Newest seen sample age\nThe age of the newest received sample seen in the distributors.\n\n",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "time() - max(cortex_distributor_latest_seen_sample_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "age",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Newest seen sample age",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Oldest exemplar age\nThe age of the oldest exemplar stored in circular storage.\nUseful to check for what time range the current exemplar buffer limit allows.\nThis usually means the max age for all exemplars for a typical setup.\nThis is not true though if one of the series timestamp is in future compared to rest series.\n\n",
                      "fill": 1,
                      "id": 7,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "time() - min(cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "age",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Oldest exemplar age",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Samples and exemplars",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor requests incoming rate\nThe rate of requests that have come in to the distributor, including rejected requests.\n\n",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_distributor_requests_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor requests incoming rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor requests received (accepted) rate\nThe rate of received requests, excluding rejected requests.\n\n",
                      "fill": 1,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "limit",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_distributor_received_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"request_rate\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"request_rate\"})\n",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor requests received (accepted) rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor discarded requests rate\nThe rate of each request's discarding reason.\n\n",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (reason) (rate(cortex_discarded_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{ reason }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor discarded requests rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Distributor ingestion requests",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor samples incoming rate\nThe rate of samples that have come in to the distributor, including rejected or deduped exemplars.\n\n",
                      "fill": 1,
                      "id": 11,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_distributor_samples_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor samples incoming rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor samples received (accepted) rate\nThe rate of received samples, excluding rejected and deduped samples.\n\n",
                      "fill": 1,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "limit",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ingestion_rate\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ingestion_rate\"})\n",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor samples received (accepted) rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor deduplicated/non-HA\nThe rate of deduplicated samples and the rate of received samples for a user that has HA tracking turned on, but the sample didn't contain both HA labels.\n\n",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_distributor_deduped_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "deduplicated",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_distributor_non_ha_samples_received_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "non-HA",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor deduplicated/non-HA",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor and ingester discarded samples rate\nThe rate of each sample's discarding reason.\n\n",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (reason) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{ reason }} (distributor)",
                            "legendLink": null
                         },
                         {
                            "expr": "sum by (reason) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{ reason }} (ingester)",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor and ingester discarded samples rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Samples ingestion funnel",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor exemplars incoming rate\nThe rate of exemplars that have come in to the distributor, including rejected or deduped exemplars.\n\n",
                      "fill": 1,
                      "id": 15,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_distributor_exemplars_in_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor exemplars incoming rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor exemplars received (accepted) rate\nThe rate of received exemplars, excluding rejected and deduped exemplars.\nThis number can be sensibly lower than incoming rate because we dedupe the HA sent exemplars, and then reject based on time.\nSee discarded rate for reasons why exemplars are being discarded.\n\n",
                      "fill": 1,
                      "id": 16,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_distributor_received_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor exemplars received (accepted) rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor discarded exemplars rate\nThe rate of each exmplars' discarding reason.\n\n",
                      "fill": 1,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (reason) (rate(cortex_discarded_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{ reason }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor discarded exemplars rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Ingester appended exemplars rate\nTotal number of exemplars appended in the ingesters.\nThis can be lower than ingested exemplars rate since TSDB does not append the same exemplar twice, and those can be frequent.\n\n",
                      "fill": 1,
                      "id": 18,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  rate(cortex_ingester_tsdb_exemplar_exemplars_appended_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"}[$__rate_interval])\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Ingester appended exemplars rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Exemplars ingestion funnel",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Symbol table size for loaded blocks\nSize of symbol table in memory for loaded blocks, averaged by ingester.\n\n",
                      "fill": 1,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (job) (cortex_ingester_tsdb_symbol_table_size_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"})",
                            "format": "time_series",
                            "legendFormat": "{{ job }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Symbol table size for loaded blocks",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Space used by local blocks\nThe number of bytes that are currently used for local storage by all blocks.\n\n",
                      "fill": 1,
                      "id": 20,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (job) (cortex_ingester_tsdb_storage_blocks_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", user=\"$user\"})",
                            "format": "time_series",
                            "legendFormat": "{{ job }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Space used by local blocks",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingesters' storage",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Number of groups\nTotal number of rule groups for a tenant.\n\n",
                      "fill": 1,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "limit",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "count(sum by (rule_group) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
                            "format": "time_series",
                            "legendFormat": "groups",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ruler_max_rule_groups_per_tenant\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/((overrides-exporter|mimir-backend.*))\", limit_name=\"ruler_max_rule_groups_per_tenant\"})\n",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Number of groups",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Number of rules\nTotal number of rules for a tenant.\n\n",
                      "fill": 1,
                      "id": 22,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
                            "format": "time_series",
                            "legendFormat": "rules",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Number of rules",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 23,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Total evaluations rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 24,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) > 0",
                            "format": "time_series",
                            "legendFormat": "{{ rule_group }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Failed evaluations rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Rules",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 25,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 2,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "rules",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit, sum by (rule_group) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit biggest groups",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 26,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 2,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "seconds",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit, sum by (rule_group) (cortex_prometheus_rule_group_last_duration_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}))",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit slowest groups (last evaluation)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Top rules",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 27,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_prometheus_notifications_sent_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Sent notifications rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "rate": "#E24D42"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 28,
                      "legend": {
                         "show": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_prometheus_notifications_errors_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "rate",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Failed notifications rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Notifications",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 29,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (user) (cortex_alertmanager_alerts{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
                            "format": "time_series",
                            "legendFormat": "alerts",
                            "legendLink": null
                         },
                         {
                            "expr": "sum by (user) (cortex_alertmanager_silences{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"})",
                            "format": "time_series",
                            "legendFormat": "silences",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Alerts",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 30,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "(\nsum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))\n-\non() (sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) or on () vector(0))\n) > 0\n",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "NPS",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 31,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "(\nsum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration)\n-\n(sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration) or\n (sum(rate(cortex_alertmanager_notifications_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration) * 0)\n)) > 0\n",
                            "format": "time_series",
                            "legendFormat": "success - {{ integration }}",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_alertmanager_notifications_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\", user=\"$user\"}[$__rate_interval])) by(integration)",
                            "format": "time_series",
                            "legendFormat": "failed - {{ integration }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "NPS by integration",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Alertmanager",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 32,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Queries / Sec",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Rate of Read Requests - query-frontend",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 33,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((query-scheduler.*|mimir-backend.*))\", user=\"$user\"})",
                            "format": "time_series",
                            "legendFormat": "Queue Length",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Number of Queries Queued - query-scheduler",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Read Path - Queries (User)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 34,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_query_frontend_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", user=\"$user\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Queries / Sec",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Rate of Read Requests - ruler-query-frontend",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 35,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cortex_query_scheduler_queue_length{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-scheduler.*))\", user=\"$user\"})",
                            "format": "time_series",
                            "legendFormat": "Queue Length",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Number of Queries Queued - ruler-query-scheduler",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Read Path - Queries (Ruler)",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "user",
                   "multi": false,
                   "name": "user",
                   "options": [ ],
                   "query": "label_values(cortex_ingester_active_series{cluster=~\"$cluster\", namespace=~\"$namespace\"}, user)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "current": {
                      "selected": true,
                      "text": "10",
                      "value": "10"
                   },
                   "hide": 0,
                   "includeAll": false,
                   "multi": false,
                   "name": "limit",
                   "options": [
                      {
                         "selected": true,
                         "text": "10",
                         "value": "10"
                      },
                      {
                         "selected": false,
                         "text": "50",
                         "value": "50"
                      },
                      {
                         "selected": false,
                         "text": "100",
                         "value": "100"
                      },
                      {
                         "selected": false,
                         "text": "500",
                         "value": "500"
                      },
                      {
                         "selected": false,
                         "text": "1000",
                         "value": "1000"
                      }
                   ],
                   "type": "custom"
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Tenants",
          "uid": "35fa247ce651ba189debf33d7ae41611",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-top-tenants-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-top-tenants.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "25px",
                "panels": [
                   {
                      "content": "<p>\n  This dashboard shows the top tenants based on multiple selection criterias.\n  Rows are collapsed by default to avoid querying all of them.\n  Use the templating variable \"limit\" above to select the amount of users to be shown.\n</p>\n",
                      "datasource": null,
                      "description": "",
                      "id": 1,
                      "mode": "markdown",
                      "span": 12,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Top tenants dashboard description",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 2,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "series",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit,\n  sum by (user) (\n    cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n    / on(cluster, namespace) group_left\n    max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n  )\n)\n",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by active series",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By active series",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 2,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "series",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit, sum by (user) (\n  (\n      sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n      -\n      sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n  )\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} )\n)\n)",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by in-memory series (series created - series removed)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By in-memory series",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (user) (\n  (\n      sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n      -\n      sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} )\n  )\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} )\n)\n\nand\ntopk($limit, sum by (user) (\n  (\n      sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ end())\n      -\n      sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ end())\n  )\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} @ end())\n)\n - sum by (user) (\n  (\n      sum by (user, cluster, namespace) (cortex_ingester_memory_series_created_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ start())\n      -\n      sum by (user, cluster, namespace) (cortex_ingester_memory_series_removed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"} @ start())\n  )\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"} @ start())\n)\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by in-memory series (series created - series removed) that grew the most between query range start and query range end",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By in-memory series growth",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 2,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "samples/s",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit, sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by received samples rate in last 5m",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By samples rate",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\nand\ntopk($limit,\n  sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ end()))\n  -\n  sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ start()))\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by received samples rate that grew the most between query range start and query range end",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By samples rate growth",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 2,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "samples/s",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit, sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by discarded samples rate in last 5m",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By discarded samples rate",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\nand\ntopk($limit,\n  sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ end()))\n  -\n  sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*|distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval] @ start()))\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{ user }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by discarded samples rate that grew the most between query range start and query range end",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By discarded samples rate growth",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 2,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "series",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit,\n  sum by (user) (\n    cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n    / on(cluster, namespace) group_left\n    max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n  )\n)\n",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by series with exemplars",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By series with exemplars",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 2,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "exemplars/s",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit, sum by (user) (rate(cortex_distributor_received_exemplars_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[5m])))",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit users by received exemplars rate in last 5m",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By exemplars rate",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 3,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "rules",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit, sum by (rule_group, user) (cortex_prometheus_rule_group_rules{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}))",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit biggest groups",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By rule group size",
                "titleSize": "h6"
             },
             {
                "collapse": true,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "sort": {
                         "col": 3,
                         "desc": true
                      },
                      "spaceLength": 10,
                      "span": 12,
                      "stack": false,
                      "steppedLine": false,
                      "styles": [
                         {
                            "alias": "Time",
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "pattern": "Time",
                            "type": "hidden"
                         },
                         {
                            "alias": "seconds",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "link": false,
                            "linkTargetBlank": false,
                            "linkTooltip": "Drill down",
                            "linkUrl": "",
                            "pattern": "Value #A",
                            "thresholds": [ ],
                            "type": "number",
                            "unit": "short"
                         },
                         {
                            "alias": "",
                            "colorMode": null,
                            "colors": [ ],
                            "dateFormat": "YYYY-MM-DD HH:mm:ss",
                            "decimals": 2,
                            "pattern": "/.*/",
                            "thresholds": [ ],
                            "type": "string",
                            "unit": "short"
                         }
                      ],
                      "targets": [
                         {
                            "expr": "topk($limit, sum by (rule_group, user) (cortex_prometheus_rule_group_last_duration_seconds{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir|mimir-backend.*))\"}))",
                            "format": "table",
                            "instant": true,
                            "legendFormat": "",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Top $limit slowest groups (last evaluation)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "transform": "table",
                      "type": "table",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "By rule group evaluation time",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "current": {
                      "selected": true,
                      "text": "10",
                      "value": "10"
                   },
                   "hide": 0,
                   "includeAll": false,
                   "multi": false,
                   "name": "limit",
                   "options": [
                      {
                         "selected": true,
                         "text": "10",
                         "value": "10"
                      },
                      {
                         "selected": false,
                         "text": "50",
                         "value": "50"
                      },
                      {
                         "selected": false,
                         "text": "100",
                         "value": "100"
                      }
                   ],
                   "type": "custom"
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Top tenants",
          "uid": "bc6e12d4fe540e4a1785b9d3ca0ffdd9",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-writes-networking-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-writes-networking.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?(distributor|ingester|mimir-write).*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Summary",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?distributor.*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Distributor",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Receive bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Transmit bandwidth",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Inflight requests (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}))",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         },
                         {
                            "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"}))",
                            "format": "time_series",
                            "legendFormat": "highest",
                            "legendLink": null
                         },
                         {
                            "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*mimir-)?ingester.*\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "TCP connections (per pod)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Writes networking",
          "uid": "978c1cb452585c96697a238eaac7fe2d",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-writes-resources-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-writes-resources.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 1,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor|ingester|mimir-write\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Summary",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Distributor",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "In-memory series",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"cpu\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "CPU",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 9,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_rss{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (RSS)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "request",
                            "color": "#FFC000",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "limit",
                            "color": "#E02F44",
                            "dashLength": 5,
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         },
                         {
                            "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"} > 0)",
                            "format": "time_series",
                            "legendFormat": "limit",
                            "legendLink": null
                         },
                         {
                            "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\",resource=\"memory\"})",
                            "format": "time_series",
                            "legendFormat": "request",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (workingset)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ingester\"})",
                            "format": "time_series",
                            "legendFormat": "{{pod}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Memory (go heap inuse)",
                      "tooltip": {
                         "sort": 2
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "bytes",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 12,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_written_bytes_total[$__rate_interval]\n  )\n)\n+\nignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"ingester\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk writes",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(instance, pod, device) (\n  rate(\n    node_disk_read_bytes_total[$__rate_interval]\n  )\n) + ignoring(pod) group_right() (\n  label_replace(\n    count by(\n      instance,\n      pod,\n      device\n    )\n    (\n      container_fs_writes_bytes_total{\n        cluster=~\"$cluster\", namespace=~\"$namespace\",\n        container=~\"ingester\",\n        device!~\".*sda.*\"\n      }\n    ),\n    \"device\",\n    \"$1\",\n    \"device\",\n    \"/dev/(.*)\"\n  ) * 0\n)\n\n",
                            "format": "time_series",
                            "legendFormat": "{{pod}} - {{device}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk reads",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "Bps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 0,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by(persistentvolumeclaim) (\n  kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} /\n  kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\nand\ncount by(persistentvolumeclaim) (\n  kube_persistentvolumeclaim_labels{\n    cluster=~\"$cluster\", namespace=~\"$namespace\",\n    label_name=~\"(ingester).*\"\n  }\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{persistentvolumeclaim}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Disk space utilization",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "percentunit",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".*",
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": false,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": null,
                   "current": {
                      "text": "prod",
                      "value": "prod"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": false,
                   "label": "namespace",
                   "multi": false,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Writes resources",
          "uid": "bc9160e50b52e89e0e49c840fea3d379",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-dashboards.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: mimir-writes-dashboard
  namespace: default
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    grafana_dashboard: "1"
  annotations:
    k8s-sidecar-target-directory: /tmp/dashboards/Mimir Dashboards
data:
  mimir-writes.json: |-
    {
          "__requires": [
             {
                "id": "grafana",
                "name": "Grafana",
                "type": "grafana",
                "version": "8.0.0"
             }
          ],
          "annotations": {
             "list": [ ]
          },
          "editable": true,
          "gnetId": null,
          "graphTooltip": 1,
          "hideControls": false,
          "links": [
             {
                "asDropdown": true,
                "icon": "external link",
                "includeVars": true,
                "keepTime": true,
                "tags": [
                   "mimir"
                ],
                "targetBlank": false,
                "title": "Mimir dashboards",
                "type": "dashboards"
             }
          ],
          "refresh": "10s",
          "rows": [
             {
                "collapse": false,
                "height": "125px",
                "panels": [
                   {
                      "content": "<p>\n  This dashboard shows various health metrics for the write path.\n  It is broken into sections for each service on the write path,\n  and organized by the order in which the write request flows.\n  <br/>\n  Incoming metrics data travels from the gateway → distributor → ingester.\n  <br/>\n  For each service, there are 3 panels showing\n  (1) requests per second to that service,\n  (2) average, median, and p99 latency of requests to that service, and\n  (3) p99 latency of requests to each instance of that service.\n</p>\n<p>\n  It also includes metrics for the key-value (KV) stores used to manage\n  the high-availability tracker and the ingesters.\n</p>\n",
                      "datasource": null,
                      "description": "",
                      "id": 1,
                      "mode": "markdown",
                      "span": 12,
                      "title": "",
                      "transparent": true,
                      "type": "text"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Writes dashboard description",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "100px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "format": "short",
                      "id": 2,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Samples / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Exemplars / sec\nThe total number of received exemplars by the distributors, excluding rejected and deduped exemplars, but not necessarily ingested by the ingesters.\n\n",
                      "fill": 1,
                      "format": "short",
                      "id": 3,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Exemplars / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### In-memory series\nThe number of series not yet flushed to object storage that are held in ingester memory.\n\n",
                      "fill": 1,
                      "format": "short",
                      "id": 4,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cortex_ingester_memory_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}))\n",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "In-memory series",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Exemplars in ingesters\nNumber of TSDB exemplars currently in ingesters' storage.\n\n",
                      "fill": 1,
                      "format": "short",
                      "id": 5,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cortex_ingester_tsdb_exemplar_exemplars_in_storage{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}))\n",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Exemplars in ingesters",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "format": "short",
                      "id": 6,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 2,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "count(count by(user) (cortex_ingester_active_series{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}))",
                            "format": "time_series",
                            "instant": true,
                            "refId": "A"
                         }
                      ],
                      "thresholds": "70,80",
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Tenants",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "singlestat",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": false,
                "title": "Headlines",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Requests / sec\nThe rate of successful, failed and rejected requests to distributor.\nRejected requests are requests that distributor fails to handle because of distributor instance limits.\nWhen distributor is configured to use \"early\" request rejection, then rejected requests are NOT included in other metrics.\nWhen distributor is not configured to use \"early\" request rejection, then rejected requests are also counted as \"errors\".\n\n",
                      "fill": 10,
                      "id": 7,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 8,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "custom": {
                               "drawStyle": "line",
                               "fillOpacity": 0,
                               "lineWidth": 1,
                               "pointSize": 5,
                               "showPoints": "never",
                               "spanNulls": false,
                               "stacking": {
                                  "group": "A",
                                  "mode": "none"
                               }
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [ ]
                      },
                      "id": 9,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "displayMode": "hidden",
                            "showLegend": false
                         },
                         "tooltip": {
                            "mode": "multi",
                            "sort": "desc"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "exemplar": true,
                            "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push|otlp_v1_metrics\"}[$__rate_interval])))",
                            "format": "time_series",
                            "legendFormat": "",
                            "legendLink": null
                         }
                      ],
                      "title": "Per pod p99 latency",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Distributor",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Requests / sec\nThe rate of successful, failed and rejected requests to ingester.\nRejected requests are requests that ingester fails to handle because of ingester instance limits (ingester-max-inflight-push-requests, ingester-max-inflight-push-requests-bytes, ingester-max-ingestion-rate).\nWhen ingester is configured to use \"early\" request rejection, then rejected requests are NOT included in other metrics.\nWhen ingester is not configured to use \"early\" request rejection, then rejected requests are also counted as \"errors\".\n\n",
                      "fill": 10,
                      "id": 10,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 11,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 4,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th percentile",
                            "refId": "A",
                            "step": 10
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th percentile",
                            "refId": "B",
                            "step": 10
                         },
                         {
                            "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"})",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C",
                            "step": 10
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "fieldConfig": {
                         "defaults": {
                            "custom": {
                               "drawStyle": "line",
                               "fillOpacity": 0,
                               "lineWidth": 1,
                               "pointSize": 5,
                               "showPoints": "never",
                               "spanNulls": false,
                               "stacking": {
                                  "group": "A",
                                  "mode": "none"
                               }
                            },
                            "thresholds": {
                               "mode": "absolute",
                               "steps": [ ]
                            },
                            "unit": "s"
                         },
                         "overrides": [ ]
                      },
                      "id": 12,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "displayMode": "hidden",
                            "showLegend": false
                         },
                         "tooltip": {
                            "mode": "multi",
                            "sort": "desc"
                         }
                      },
                      "span": 4,
                      "targets": [
                         {
                            "exemplar": true,
                            "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))",
                            "format": "time_series",
                            "legendFormat": "",
                            "legendLink": null
                         }
                      ],
                      "title": "Per pod p99 latency",
                      "type": "timeseries"
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Replicas\nThe maximum and current number of distributor replicas.\nNote: The current number of replicas can still show 1 replica even when scaled to 0.\nBecause HPA never reports 0 replicas, the query will report 0 only if the HPA is not active.\n\n",
                      "fill": 1,
                      "id": 13,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [
                         {
                            "alias": "/Max .+/",
                            "dashes": true,
                            "fill": 0
                         },
                         {
                            "alias": "/Current .+/",
                            "fill": 0
                         },
                         {
                            "alias": "/Min .+/",
                            "dashes": true,
                            "fill": 0
                         }
                      ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n  # Add the scaletargetref_name label for readability\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Max {{ scaletargetref_name }}",
                            "legendLink": null
                         },
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n  # HPA doesn't go to 0 replicas, so we multiply by 0 if the HPA is not active\n  * on (cluster, namespace, horizontalpodautoscaler)\n    kube_horizontalpodautoscaler_status_condition{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\", condition=\"ScalingActive\", status=\"true\"}\n  # Add the scaletargetref_name label for readability\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Current {{ scaletargetref_name }}",
                            "legendLink": null
                         },
                         {
                            "expr": "max by (scaletargetref_name) (\n  kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n  # Add the scaletargetref_name label for readability\n  + on (cluster, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n    0*kube_horizontalpodautoscaler_info{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"}\n)\n",
                            "format": "time_series",
                            "legendFormat": "Min {{ scaletargetref_name }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Replicas",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Scaling metric (CPU): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
                      "fill": 1,
                      "id": 14,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (scaler) (\n  label_replace(\n    keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n    \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n  )\n  /\n  on(cluster, namespace, scaledObject, metric) group_left label_replace(\n    label_replace(\n      kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"},\n      \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n    ),\n    \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n  )\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{ scaler }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Scaling metric (CPU): Desired replicas",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Scaling metric (memory): Desired replicas\nThis panel shows the scaling metric exposed by KEDA divided by the target/threshold used.\nIt should represent the desired number of replicas, ignoring the min/max constraints applied later.\n\n",
                      "fill": 1,
                      "id": 15,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (scaler) (\n  label_replace(\n    keda_scaler_metrics_value{cluster=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n    \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n  )\n  /\n  on(cluster, namespace, scaledObject, metric) group_left label_replace(\n    label_replace(\n      kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"},\n      \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n    ),\n    \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n  )\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{ scaler }}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Scaling metric (memory): Desired replicas",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n",
                      "fill": 1,
                      "id": 16,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by(cluster, namespace, scaler, metric, scaledObject) (\n  label_replace(\n    rate(keda_scaler_errors[$__rate_interval]),\n    \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n  )\n) +\non(cluster, namespace, metric, scaledObject) group_left\nlabel_replace(\n  label_replace(\n      kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"keda-hpa-distributor\"} * 0,\n      \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"keda-hpa-(.*)\"\n  ),\n  \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n",
                            "format": "time_series",
                            "legendFormat": "{{scaler}} failures",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Autoscaler failures rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Distributor - autoscaling",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 17,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 18,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-hatracker\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Distributor - key-value store for high-availability (HA) deduplication",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 19,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 20,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\", kv_name=~\"distributor-(lifecycler|ring)\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Distributor - key-value store for distributors ring",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "1xx": "#EAB839",
                         "2xx": "#7EB26D",
                         "3xx": "#6ED0E0",
                         "4xx": "#EF843C",
                         "5xx": "#E24D42",
                         "OK": "#7EB26D",
                         "cancel": "#A9A9A9",
                         "error": "#E24D42",
                         "success": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 21,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (status) (\n  label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval]),\n  \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n  \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n",
                            "format": "time_series",
                            "legendFormat": "{{status}}",
                            "refId": "A"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Requests / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "reqps",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 22,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_kv_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_kv_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_kv_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\", kv_name=~\"ingester-.*\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester - key-value store for the ingesters ring",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Uploaded blocks / sec\nThe rate of blocks being uploaded from the ingesters\nto object storage.\n\n",
                      "fill": 10,
                      "id": 23,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_ingester_shipper_uploads_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_ingester_shipper_upload_failures_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Uploaded blocks / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Upload latency\nThe average, median (50th percentile), and 99th percentile time\nthe ingesters take to upload blocks to object storage.\n\n",
                      "fill": 1,
                      "id": 24,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\",component=\"ingester\",operation=\"upload\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Upload latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester - shipper",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Compactions per second\nIngesters maintain a local TSDB per-tenant on disk. Each TSDB maintains a head block for each\nactive time series; these blocks get periodically compacted (by default, every 2h).\nThis panel shows the rate of compaction operations across all TSDBs on all ingesters.\n\n",
                      "fill": 10,
                      "id": 25,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_compactions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_compactions_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Compactions / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Compaction latency\nThe average, median (50th percentile), and 99th percentile time ingesters take to compact TSDB head blocks\non the local filesystem.\n\n",
                      "fill": 1,
                      "id": 26,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "99th Percentile",
                            "refId": "A"
                         },
                         {
                            "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) by (le)) * 1e3",
                            "format": "time_series",
                            "legendFormat": "50th Percentile",
                            "refId": "B"
                         },
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "Average",
                            "refId": "C"
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Compactions latency",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ms",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester - TSDB head",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### WAL truncations per second\nThe WAL is truncated each time a new TSDB block is written. This panel measures the rate of\ntruncations.\n\n",
                      "fill": 10,
                      "id": 27,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "WAL truncations / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": {
                         "failed": "#E24D42",
                         "successful": "#7EB26D"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Checkpoints created per second\nCheckpoints are created as part of the WAL truncation process.\nThis metric measures the rate of checkpoint creation.\n\n",
                      "fill": 10,
                      "id": 28,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "successful",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "failed",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Checkpoints created / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "datasource": "$datasource",
                      "description": "### WAL truncations latency (including checkpointing)\nAverage time taken to perform a full WAL truncation,\nincluding the time taken for the checkpointing to complete.\n\n",
                      "fieldConfig": {
                         "defaults": {
                            "noValue": "0",
                            "unit": "s"
                         }
                      },
                      "id": 29,
                      "links": [ ],
                      "options": {
                         "legend": {
                            "showLegend": true
                         },
                         "tooltip": {
                            "mode": "single",
                            "sort": "none"
                         }
                      },
                      "span": 3,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))\n/\nsum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval])) >= 0\n",
                            "format": "time_series",
                            "legendFormat": "avg",
                            "legendLink": null
                         }
                      ],
                      "title": "WAL truncations latency (includes checkpointing)",
                      "type": "timeseries"
                   },
                   {
                      "aliasColors": {
                         "WAL": "#E24D42",
                         "mmap-ed chunks": "#E28A42"
                      },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 10,
                      "id": 30,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 0,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": true,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_wal_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "WAL",
                            "legendLink": null
                         },
                         {
                            "expr": "sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "mmap-ed chunks",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Corruptions / sec",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ops",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Ingester - TSDB write ahead log (WAL)",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor exemplars incoming rate\nThe rate of exemplars that have come in to the distributor, including rejected or deduped exemplars.\n\n",
                      "fill": 1,
                      "id": 31,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_namespace_job:cortex_distributor_exemplars_in:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "legendFormat": "incoming exemplars",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor exemplars incoming rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ex/s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Distributor exemplars received rate\nThe rate of received exemplars, excluding rejected and deduped exemplars.\nThis number can be sensibly lower than incoming rate because we dedupe the HA sent exemplars, and then reject based on time, see `cortex_discarded_exemplars_total` for specific reasons rates.\n\n",
                      "fill": 1,
                      "id": 32,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(cluster_namespace_job:cortex_distributor_received_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})",
                            "format": "time_series",
                            "legendFormat": "received exemplars",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Distributor exemplars received rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ex/s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Ingester ingested exemplars rate\nThe rate of exemplars ingested in the ingesters.\nEvery exemplar is sent to the replication factor number of ingesters, so the sum of rates from all ingesters is divided by the replication factor.\nThis ingested exemplars rate should match the distributor's received exemplars rate.\n\n",
                      "fill": 1,
                      "id": 33,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  cluster_namespace_job:cortex_ingester_ingested_exemplars:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
                            "format": "time_series",
                            "legendFormat": "ingested exemplars",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Ingester ingested exemplars rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ex/s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "description": "### Ingester appended exemplars rate\nThe rate of exemplars appended in the ingesters.\nThis can be lower than ingested exemplars rate since TSDB does not append the same exemplar twice, and those can be frequent.\n\n",
                      "fill": 1,
                      "id": 34,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 3,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum(\n  cluster_namespace_job:cortex_ingester_tsdb_exemplar_exemplars_appended:rate5m{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}\n  / on(cluster, namespace) group_left\n  max by (cluster, namespace) (cortex_distributor_replication_factor{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"})\n)\n",
                            "format": "time_series",
                            "legendFormat": "appended exemplars",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Ingester appended exemplars rate",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "ex/s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Exemplars",
                "titleSize": "h6"
             },
             {
                "collapse": false,
                "height": "250px",
                "panels": [
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 35,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (reason) (rate(cortex_distributor_instance_rejected_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{reason}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Rejected distributor requests",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "req/s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   },
                   {
                      "aliasColors": { },
                      "bars": false,
                      "dashLength": 10,
                      "dashes": false,
                      "datasource": "$datasource",
                      "fill": 1,
                      "id": 36,
                      "legend": {
                         "avg": false,
                         "current": false,
                         "max": false,
                         "min": false,
                         "show": true,
                         "total": false,
                         "values": false
                      },
                      "lines": true,
                      "linewidth": 1,
                      "links": [ ],
                      "nullPointMode": "null as zero",
                      "percentage": false,
                      "pointradius": 5,
                      "points": false,
                      "renderer": "flot",
                      "seriesOverrides": [ ],
                      "spaceLength": 10,
                      "span": 6,
                      "stack": false,
                      "steppedLine": false,
                      "targets": [
                         {
                            "expr": "sum by (reason) (rate(cortex_ingester_instance_rejected_requests_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir|mimir-write.*))\"}[$__rate_interval]))",
                            "format": "time_series",
                            "legendFormat": "{{reason}}",
                            "legendLink": null
                         }
                      ],
                      "thresholds": [ ],
                      "timeFrom": null,
                      "timeShift": null,
                      "title": "Rejected ingester requests",
                      "tooltip": {
                         "shared": false,
                         "sort": 0,
                         "value_type": "individual"
                      },
                      "type": "graph",
                      "xaxis": {
                         "buckets": null,
                         "mode": "time",
                         "name": null,
                         "show": true,
                         "values": [ ]
                      },
                      "yaxes": [
                         {
                            "format": "req/s",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                         },
                         {
                            "format": "short",
                            "label": null,
                            "logBase": 1,
                            "max": null,
                            "min": null,
                            "show": false
                         }
                      ]
                   }
                ],
                "repeat": null,
                "repeatIteration": null,
                "repeatRowId": null,
                "showTitle": true,
                "title": "Instance Limits",
                "titleSize": "h6"
             }
          ],
          "schemaVersion": 14,
          "style": "dark",
          "tags": [
             "mimir"
          ],
          "templating": {
             "list": [
                {
                   "current": {
                      "text": "default",
                      "value": "default"
                   },
                   "hide": 0,
                   "label": "Data source",
                   "name": "datasource",
                   "options": [ ],
                   "query": "prometheus",
                   "refresh": 1,
                   "regex": "",
                   "type": "datasource"
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "cluster",
                   "multi": true,
                   "name": "cluster",
                   "options": [ ],
                   "query": "label_values(cortex_build_info, cluster)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                },
                {
                   "allValue": ".+",
                   "current": {
                      "selected": true,
                      "text": "All",
                      "value": "$__all"
                   },
                   "datasource": "$datasource",
                   "hide": 0,
                   "includeAll": true,
                   "label": "namespace",
                   "multi": true,
                   "name": "namespace",
                   "options": [ ],
                   "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)",
                   "refresh": 1,
                   "regex": "",
                   "sort": 1,
                   "tagValuesQuery": "",
                   "tags": [ ],
                   "tagsQuery": "",
                   "type": "query",
                   "useTags": false
                }
             ]
          },
          "time": {
             "from": "now-1h",
             "to": "now"
          },
          "timepicker": {
             "refresh_intervals": [
                "5s",
                "10s",
                "30s",
                "1m",
                "5m",
                "15m",
                "30m",
                "1h",
                "2h",
                "1d"
             ],
             "time_options": [
                "5m",
                "15m",
                "1h",
                "6h",
                "12h",
                "24h",
                "2d",
                "7d",
                "30d"
             ]
          },
          "timezone": "utc",
          "title": "Mimir / Writes",
          "uid": "8280707b8f16e7b87b840fc1cc92d4c5",
          "version": 0
       }
---
# Source: mimir/charts/mimir/templates/mimir-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-mimir-config
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
data:
  mimir.yaml: |
    
    activity_tracker:
      filepath: /active-query-tracker/activity.log
    alertmanager:
      data_dir: /data
      enable_api: true
      external_url: /alertmanager
      fallback_config_file: /configs/alertmanager_fallback_config.yaml
    alertmanager_storage:
      backend: s3
      s3:
        access_key_id: grafana-mimir
        bucket_name: mimir-ruler
        endpoint: release-name-minio.default.svc:9000
        insecure: true
        secret_access_key: supersecret
    blocks_storage:
      backend: s3
      bucket_store:
        index_cache:
          backend: memcached
          memcached:
            addresses: dns+release-name-mimir-index-cache.default.svc:11211
            max_idle_connections: 150
            max_item_size: 5242880
            timeout: 450ms
        metadata_cache:
          backend: memcached
          memcached:
            addresses: dns+release-name-mimir-metadata-cache.default.svc:11211
            max_idle_connections: 150
            max_item_size: 1048576
        sync_dir: /data/tsdb-sync
      s3:
        access_key_id: grafana-mimir
        bucket_name: mimir-tsdb
        endpoint: release-name-minio.default.svc:9000
        insecure: true
        secret_access_key: supersecret
      tsdb:
        dir: /data/tsdb
        head_compaction_interval: 15m
        wal_replay_concurrency: 3
    compactor:
      compaction_interval: 30m
      data_dir: /data
      deletion_delay: 2h
      first_level_compaction_wait_period: 25m
      max_closing_blocks_concurrency: 2
      max_opening_blocks_concurrency: 4
      sharding_ring:
        wait_stability_min_duration: 1m
      symbols_flushers_concurrency: 4
    frontend:
      cache_results: true
      parallelize_shardable_queries: true
      query_sharding_target_series_per_shard: 2500
      results_cache:
        backend: memcached
        memcached:
          addresses: dns+release-name-mimir-results-cache.default.svc:11211
          max_item_size: 5242880
          timeout: 500ms
      scheduler_address: release-name-mimir-query-scheduler-headless.default.svc:9095
    frontend_worker:
      grpc_client_config:
        max_send_msg_size: 419430400
      scheduler_address: release-name-mimir-query-scheduler-headless.default.svc:9095
    ingester:
      ring:
        final_sleep: 0s
        num_tokens: 512
        tokens_file_path: /data/tokens
        unregister_on_shutdown: false
        zone_awareness_enabled: true
    ingester_client:
      grpc_client_config:
        max_recv_msg_size: 104857600
        max_send_msg_size: 104857600
    limits:
      max_cache_freshness: 10m
      max_query_parallelism: 240
      max_total_query_length: 12000h
    memberlist:
      abort_if_cluster_join_fails: false
      compression_enabled: false
      join_members:
      - dns+release-name-mimir-gossip-ring.default.svc.cluster.local.:7946
    querier:
      max_concurrent: 16
    query_scheduler:
      max_outstanding_requests_per_tenant: 800
    ruler:
      alertmanager_url: dnssrvnoa+http://_http-metrics._tcp.release-name-mimir-alertmanager-headless.default.svc.cluster.local./alertmanager
      enable_api: true
      rule_path: /data
    ruler_storage:
      backend: s3
      cache:
        backend: memcached
        memcached:
          addresses: dns+release-name-mimir-metadata-cache.default.svc:11211
          max_item_size: 1048576
      s3:
        access_key_id: grafana-mimir
        bucket_name: mimir-ruler
        endpoint: release-name-minio.default.svc:9000
        insecure: true
        secret_access_key: supersecret
    runtime_config:
      file: /var/mimir/runtime.yaml
    server:
      grpc_server_max_connection_age: 2m
      grpc_server_max_connection_age_grace: 5m
      grpc_server_max_connection_idle: 1m
    store_gateway:
      sharding_ring:
        kvstore:
          prefix: multi-zone/
        tokens_file_path: /data/tokens
        unregister_on_shutdown: false
        wait_stability_min_duration: 1m
        zone_awareness_enabled: true
    usage_stats:
      installation_mode: helm
---
# Source: mimir/charts/mimir/templates/nginx/nginx-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-mimir-nginx
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: nginx
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
data:
  nginx.conf: |
    worker_processes  5;  ## Default: 1
    error_log  /dev/stderr error;
    pid        /tmp/nginx.pid;
    worker_rlimit_nofile 8192;
    
    events {
      worker_connections  4096;  ## Default: 1024
    }
    
    http {
      client_body_temp_path /tmp/client_temp;
      proxy_temp_path       /tmp/proxy_temp_path;
      fastcgi_temp_path     /tmp/fastcgi_temp;
      uwsgi_temp_path       /tmp/uwsgi_temp;
      scgi_temp_path        /tmp/scgi_temp;
    
      default_type application/octet-stream;
      log_format   main '$remote_addr - $remote_user [$time_local]  $status '
            '"$request" $body_bytes_sent "$http_referer" '
            '"$http_user_agent" "$http_x_forwarded_for"';
      access_log   /dev/stderr  main;
    
      sendfile     on;
      tcp_nopush   on;
      resolver kube-dns.kube-system.svc.cluster.local.;
    
      # Ensure that X-Scope-OrgID is always present, default to the no_auth_tenant for backwards compatibility when multi-tenancy was turned off.
      map $http_x_scope_orgid $ensured_x_scope_orgid {
        default $http_x_scope_orgid;
        "" "anonymous";
      }
    
      proxy_read_timeout 300;
      server {
        listen 8080;
        listen [::]:8080;
    
        location = / {
          return 200 'OK';
          auth_basic off;
        }
    
        proxy_set_header X-Scope-OrgID $ensured_x_scope_orgid;
    
        # Distributor endpoints
        location /distributor {
          set $distributor release-name-mimir-distributor-headless.default.svc.cluster.local.;
          proxy_pass      http://$distributor:8080$request_uri;
        }
        location = /api/v1/push {
          set $distributor release-name-mimir-distributor-headless.default.svc.cluster.local.;
          proxy_pass      http://$distributor:8080$request_uri;
        }
        location /otlp/v1/metrics {
          set $distributor release-name-mimir-distributor-headless.default.svc.cluster.local.;
          proxy_pass      http://$distributor:8080$request_uri;
        }
    
        # Alertmanager endpoints
        location /alertmanager {
          set $alertmanager release-name-mimir-alertmanager-headless.default.svc.cluster.local.;
          proxy_pass      http://$alertmanager:8080$request_uri;
        }
        location = /multitenant_alertmanager/status {
          set $alertmanager release-name-mimir-alertmanager-headless.default.svc.cluster.local.;
          proxy_pass      http://$alertmanager:8080$request_uri;
        }
        location = /api/v1/alerts {
          set $alertmanager release-name-mimir-alertmanager-headless.default.svc.cluster.local.;
          proxy_pass      http://$alertmanager:8080$request_uri;
        }
    
        # Ruler endpoints
        location /prometheus/config/v1/rules {
          set $ruler release-name-mimir-ruler.default.svc.cluster.local.;
          proxy_pass      http://$ruler:8080$request_uri;
        }
        location /prometheus/api/v1/rules {
          set $ruler release-name-mimir-ruler.default.svc.cluster.local.;
          proxy_pass      http://$ruler:8080$request_uri;
        }
    
        location /prometheus/api/v1/alerts {
          set $ruler release-name-mimir-ruler.default.svc.cluster.local.;
          proxy_pass      http://$ruler:8080$request_uri;
        }
        location = /ruler/ring {
          set $ruler release-name-mimir-ruler.default.svc.cluster.local.;
          proxy_pass      http://$ruler:8080$request_uri;
        }
    
        # Rest of /prometheus goes to the query frontend
        location /prometheus {
          set $query_frontend release-name-mimir-query-frontend.default.svc.cluster.local.;
          proxy_pass      http://$query_frontend:8080$request_uri;
        }
    
        # Buildinfo endpoint can go to any component
        location = /api/v1/status/buildinfo {
          set $query_frontend release-name-mimir-query-frontend.default.svc.cluster.local.;
          proxy_pass      http://$query_frontend:8080$request_uri;
        }
    
        # Compactor endpoint for uploading blocks
        location /api/v1/upload/block/ {
          set $compactor release-name-mimir-compactor.default.svc.cluster.local.;
          proxy_pass      http://$compactor:8080$request_uri;
        }
      }
    }
---
# Source: mimir/charts/mimir/templates/runtime-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-mimir-runtime
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
data:
  runtime.yaml: |
    
    {}
---
# Source: mimir/templates/common.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: release-name-mimir-grafana-datasource
  namespace: default
  labels:
    app: "mimir-0.1.8"
    app.kubernetes.io/instance: "release-name"
    app.kubernetes.io/managed-by: "Helm"
    app.kubernetes.io/name: "mimir"
    app.kubernetes.io/version: "18.0.1"
    grafana_datasources: "1"
    helm-revision: "1"
    helm.sh/chart: "mimir-0.1.8"
    release: "release-name"
data:
  datasource.yaml: |-
    apiVersion: 1
    datasources:
    - name: Mimir-Metamonitoring
      type: prometheus
      uid: meta
      url: http://release-name-mimir-nginx.default/prometheus
      access: proxy
      isDefault: false
      jsonData:
        httpMethod: POST
        timeInterval: 30s
        httpHeaderName1: 'X-Scope-OrgID'
      secureJsonData:
        httpHeaderValue1: 'metamonitoring'
    - name: Mimir
      type: prometheus
      uid: meta
      url: http://release-name-mimir-nginx.default/prometheus
      access: proxy
      isDefault: true
      jsonData:
        httpMethod: POST
        timeInterval: 30s
---
# Source: mimir/charts/mimir/charts/minio/templates/pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: release-name-minio
  labels:
    app: minio
    chart: minio-5.0.14
    release: release-name
    heritage: Helm
spec:
  accessModes:
    - "ReadWriteOnce"
  resources:
    requests:
      storage: "5Gi"
---
# Source: mimir/charts/kps/charts/kube-state-metrics/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:    
    helm.sh/chart: kube-state-metrics-5.16.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.10.1"
    release: release-name
  name: release-name-kube-state-metrics
rules:

- apiGroups: ["certificates.k8s.io"]
  resources:
  - certificatesigningrequests
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - configmaps
  verbs: ["list", "watch"]

- apiGroups: ["batch"]
  resources:
  - cronjobs
  verbs: ["list", "watch"]

- apiGroups: ["extensions", "apps"]
  resources:
  - daemonsets
  verbs: ["list", "watch"]

- apiGroups: ["extensions", "apps"]
  resources:
  - deployments
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - endpoints
  verbs: ["list", "watch"]

- apiGroups: ["autoscaling"]
  resources:
  - horizontalpodautoscalers
  verbs: ["list", "watch"]

- apiGroups: ["extensions", "networking.k8s.io"]
  resources:
  - ingresses
  verbs: ["list", "watch"]

- apiGroups: ["batch"]
  resources:
  - jobs
  verbs: ["list", "watch"]

- apiGroups: ["coordination.k8s.io"]
  resources:
  - leases
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - limitranges
  verbs: ["list", "watch"]

- apiGroups: ["admissionregistration.k8s.io"]
  resources:
    - mutatingwebhookconfigurations
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - namespaces
  verbs: ["list", "watch"]

- apiGroups: ["networking.k8s.io"]
  resources:
  - networkpolicies
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - nodes
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - persistentvolumeclaims
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - persistentvolumes
  verbs: ["list", "watch"]

- apiGroups: ["policy"]
  resources:
    - poddisruptionbudgets
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - pods
  verbs: ["list", "watch"]

- apiGroups: ["extensions", "apps"]
  resources:
  - replicasets
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - replicationcontrollers
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - resourcequotas
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - secrets
  verbs: ["list", "watch"]

- apiGroups: [""]
  resources:
  - services
  verbs: ["list", "watch"]

- apiGroups: ["apps"]
  resources:
  - statefulsets
  verbs: ["list", "watch"]

- apiGroups: ["storage.k8s.io"]
  resources:
    - storageclasses
  verbs: ["list", "watch"]

- apiGroups: ["admissionregistration.k8s.io"]
  resources:
    - validatingwebhookconfigurations
  verbs: ["list", "watch"]

- apiGroups: ["storage.k8s.io"]
  resources:
    - volumeattachments
  verbs: ["list", "watch"]
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-agent-cluster-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: release-name-mimir-grafana-agent
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: meta-monitoring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
rules:
  - apiGroups:
      - ""
    resources:
      - nodes
      - nodes/proxy
      - nodes/metrics
      - services
      - endpoints
      - pods
      - events
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - networking.k8s.io
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - nonResourceURLs:
      - /metrics
      - /metrics/cadvisor
    verbs:
      - get
---
# Source: mimir/charts/kps/charts/kube-state-metrics/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:    
    helm.sh/chart: kube-state-metrics-5.16.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.10.1"
    release: release-name
  name: release-name-kube-state-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: release-name-kube-state-metrics
subjects:
- kind: ServiceAccount
  name: release-name-kube-state-metrics
  namespace: default
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-agent-cluster-role-binding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: release-name-mimir-grafana-agent
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: meta-monitoring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: release-name-mimir-grafana-agent
subjects:
  - kind: ServiceAccount
    name: release-name-mimir
    namespace: default
---
# Source: mimir/charts/mimir/charts/rollout_operator/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: release-name-rollout-operator
rules:
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - list
  - get
  - watch
  - delete
- apiGroups:
  - apps
  resources:
  - statefulsets
  verbs:
  - list
  - get
  - watch
- apiGroups:
  - apps
  resources:
  - statefulsets/status
  verbs:
  - update
---
# Source: mimir/charts/mimir/charts/rollout_operator/templates/rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: release-name-rollout-operator
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: release-name-rollout-operator
subjects:
- kind: ServiceAccount
  name: release-name-rollout-operator
---
# Source: mimir/charts/kps/charts/kube-state-metrics/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-kube-state-metrics
  namespace: default
  labels:    
    helm.sh/chart: kube-state-metrics-5.16.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.10.1"
    release: release-name
  annotations:
    prometheus.io/scrape: 'true'
spec:
  type: "ClusterIP"
  ports:
  - name: "http"
    protocol: TCP
    port: 8080
    targetPort: 8080
  
  selector:    
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
---
# Source: mimir/charts/kps/charts/prometheus-node-exporter/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-prometheus-node-exporter
  namespace: default
  labels:
    helm.sh/chart: prometheus-node-exporter-4.30.2
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: prometheus-node-exporter
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.7.0"
    jobLabel: node-exporter
    release: release-name
  annotations:
    prometheus.io/scrape: "true"
spec:
  type: ClusterIP
  ports:
    - port: 9100
      targetPort: 9100
      protocol: TCP
      name: http-metrics
  selector:
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
---
# Source: mimir/charts/kps/templates/exporters/core-dns/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-kps-coredns
  labels:
    app: kps-coredns
    jobLabel: coredns
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
  namespace: kube-system
spec:
  clusterIP: None
  ports:
    - name: http-metrics
      port: 9153
      protocol: TCP
      targetPort: 9153
  selector:
    k8s-app: kube-dns
---
# Source: mimir/charts/kps/templates/exporters/kube-controller-manager/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-kps-kube-controller-manager
  labels:
    app: kps-kube-controller-manager
    jobLabel: kube-controller-manager
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
  namespace: kube-system
spec:
  clusterIP: None
  ports:
    - name: http-metrics
      port: 10257
      protocol: TCP
      targetPort: 10257
  selector:
    component: kube-controller-manager
  type: ClusterIP
---
# Source: mimir/charts/kps/templates/exporters/kube-etcd/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-kps-kube-etcd
  labels:
    app: kps-kube-etcd
    jobLabel: kube-etcd
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
  namespace: kube-system
spec:
  clusterIP: None
  ports:
    - name: http-metrics
      port: 2381
      protocol: TCP
      targetPort: 2381
  selector:
    component: etcd
  type: ClusterIP
---
# Source: mimir/charts/kps/templates/exporters/kube-scheduler/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-kps-kube-scheduler
  labels:
    app: kps-kube-scheduler
    jobLabel: kube-scheduler
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
  namespace: kube-system
spec:
  clusterIP: None
  ports:
    - name: http-metrics
      port: 10259
      protocol: TCP
      targetPort: 10259
  selector:
    component: kube-scheduler
  type: ClusterIP
---
# Source: mimir/charts/mimir/charts/minio/templates/console-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-minio-console
  labels:
    app: minio
    chart: minio-5.0.14
    release: release-name
    heritage: Helm
spec:
  type: ClusterIP
  ports:
    - name: http
      port: 9001
      protocol: TCP
      targetPort: 9001
  selector:
    app: minio
    release: release-name
---
# Source: mimir/charts/mimir/charts/minio/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-minio
  labels:
    app: minio
    chart: minio-5.0.14
    release: release-name
    heritage: Helm
    monitoring: "true"
spec:
  type: ClusterIP
  ports:
    - name: http
      port: 9000
      protocol: TCP
      targetPort: 9000
  selector:
    app: minio
    release: release-name
---
# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-alertmanager-headless
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: alertmanager
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    prometheus.io/service-monitor: "false"
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  clusterIP: None
  publishNotReadyAddresses: true
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
    - port: 9094
      protocol: TCP
      name: cluster
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: alertmanager
---
# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-alertmanager
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: alertmanager
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: alertmanager
---
# Source: mimir/charts/mimir/templates/compactor/compactor-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-compactor
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: compactor
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: compactor
---
# Source: mimir/charts/mimir/templates/distributor/distributor-svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-distributor-headless
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: distributor
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    prometheus.io/service-monitor: "false"
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: distributor
---
# Source: mimir/charts/mimir/templates/distributor/distributor-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-distributor
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: distributor
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: distributor
---
# Source: mimir/charts/mimir/templates/gossip-ring/gossip-ring-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-gossip-ring
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: gossip-ring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  namespace: "default"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: gossip-ring
      port: 7946
      appProtocol: tcp
      protocol: TCP
      targetPort: 7946
  publishNotReadyAddresses: true
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/part-of: memberlist
---
# Source: mimir/charts/mimir/templates/index-cache/index-cache-svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-index-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: index-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: memcached-client
      port: 11211
      targetPort: 11211
    
    - name: http-metrics
      port: 9150
      targetPort: 9150
    
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: index-cache
---
# Source: mimir/charts/mimir/templates/ingester/ingester-svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-ingester-headless
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    prometheus.io/service-monitor: "false"
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
---
# Source: mimir/charts/mimir/templates/ingester/ingester-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-ingester-zone-a
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "ingester-zone-a"
    rollout-group: ingester
    zone: zone-a
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    rollout-group: ingester
    zone: zone-a
---
# Source: mimir/charts/mimir/templates/ingester/ingester-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-ingester-zone-b
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "ingester-zone-b"
    rollout-group: ingester
    zone: zone-b
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    rollout-group: ingester
    zone: zone-b
---
# Source: mimir/charts/mimir/templates/ingester/ingester-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-ingester-zone-c
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "ingester-zone-c"
    rollout-group: ingester
    zone: zone-c
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    rollout-group: ingester
    zone: zone-c
---
# Source: mimir/charts/mimir/templates/metadata-cache/metadata-cache-svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-metadata-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: metadata-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: memcached-client
      port: 11211
      targetPort: 11211
    
    - name: http-metrics
      port: 9150
      targetPort: 9150
    
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: metadata-cache
---
# Source: mimir/charts/mimir/templates/nginx/nginx-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-nginx
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: nginx
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - name: http-metric
      port: 80
      targetPort: http-metric
      protocol: TCP
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: nginx
---
# Source: mimir/charts/mimir/templates/overrides-exporter/overrides-exporter-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-overrides-exporter
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: overrides-exporter
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: overrides-exporter
---
# Source: mimir/charts/mimir/templates/querier/querier-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-querier
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: querier
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: querier
---
# Source: mimir/charts/mimir/templates/query-frontend/query-frontend-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-query-frontend
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-frontend
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-frontend
---
# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-query-scheduler-headless
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-scheduler
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    prometheus.io/service-monitor: "false"
  annotations:
    {}
spec:
  type: ClusterIP
  clusterIP: None
  publishNotReadyAddresses: true
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-scheduler
---
# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-query-scheduler
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-scheduler
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-scheduler
---
# Source: mimir/charts/mimir/templates/results-cache/results-cache-svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-results-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: results-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - name: memcached-client
      port: 11211
      targetPort: 11211
    
    - name: http-metrics
      port: 9150
      targetPort: 9150
    
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: results-cache
---
# Source: mimir/charts/mimir/templates/ruler/ruler-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-ruler
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ruler
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ruler
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-svc-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-store-gateway-headless
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    prometheus.io/service-monitor: "false"
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  clusterIP: None
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-store-gateway-zone-a
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "store-gateway-zone-a"
    rollout-group: store-gateway
    zone: zone-a
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    rollout-group: store-gateway
    zone: zone-a
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-store-gateway-zone-b
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "store-gateway-zone-b"
    rollout-group: store-gateway
    zone: zone-b
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    rollout-group: store-gateway
    zone: zone-b
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-svc.yaml
apiVersion: v1
kind: Service
metadata:
  name: release-name-mimir-store-gateway-zone-c
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "store-gateway-zone-c"
    rollout-group: store-gateway
    zone: zone-c
  annotations:
    {}
  namespace: "default"
spec:
  type: ClusterIP
  ports:
    - port: 8080
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
    - port: 9095
      protocol: TCP
      name: grpc
      targetPort: grpc
  selector:
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    rollout-group: store-gateway
    zone: zone-c
---
# Source: mimir/charts/kps/charts/prometheus-node-exporter/templates/daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: release-name-prometheus-node-exporter
  namespace: default
  labels:
    helm.sh/chart: prometheus-node-exporter-4.30.2
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: prometheus-node-exporter
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.7.0"
    jobLabel: node-exporter
    release: release-name
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus-node-exporter
      app.kubernetes.io/instance: release-name
  revisionHistoryLimit: 10
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      annotations:
        cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
      labels:
        helm.sh/chart: prometheus-node-exporter-4.30.2
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: metrics
        app.kubernetes.io/part-of: prometheus-node-exporter
        app.kubernetes.io/name: prometheus-node-exporter
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "1.7.0"
        jobLabel: node-exporter
        release: release-name
    spec:
      automountServiceAccountToken: false
      securityContext:
        fsGroup: 65534
        runAsGroup: 65534
        runAsNonRoot: true
        runAsUser: 65534
      serviceAccountName: release-name-prometheus-node-exporter
      containers:
        - name: node-exporter
          image: quay.io/prometheus/node-exporter:v1.7.0
          imagePullPolicy: IfNotPresent
          args:
            - --path.procfs=/host/proc
            - --path.sysfs=/host/sys
            - --path.rootfs=/host/root
            - --path.udev.data=/host/root/run/udev/data
            - --web.listen-address=[$(HOST_IP)]:9100
            - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
            - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
          securityContext:
            readOnlyRootFilesystem: true
          env:
            - name: HOST_IP
              value: 0.0.0.0
          ports:
            - name: http-metrics
              containerPort: 9100
              protocol: TCP
          livenessProbe:
            failureThreshold: 3
            httpGet:
              httpHeaders:
              path: /
              port: 9100
              scheme: HTTP
            initialDelaySeconds: 0
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          readinessProbe:
            failureThreshold: 3
            httpGet:
              httpHeaders:
              path: /
              port: 9100
              scheme: HTTP
            initialDelaySeconds: 0
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          volumeMounts:
            - name: proc
              mountPath: /host/proc
              readOnly:  true
            - name: sys
              mountPath: /host/sys
              readOnly: true
            - name: root
              mountPath: /host/root
              mountPropagation: HostToContainer
              readOnly: true
      hostNetwork: true
      hostPID: true
      nodeSelector:
        kubernetes.io/os: linux
      tolerations:
        - effect: NoSchedule
          operator: Exists
      volumes:
        - name: proc
          hostPath:
            path: /proc
        - name: sys
          hostPath:
            path: /sys
        - name: root
          hostPath:
            path: /
---
# Source: mimir/charts/kps/charts/kube-state-metrics/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-kube-state-metrics
  namespace: default
  labels:    
    helm.sh/chart: kube-state-metrics-5.16.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.10.1"
    release: release-name
spec:
  selector:
    matchLabels:      
      app.kubernetes.io/name: kube-state-metrics
      app.kubernetes.io/instance: release-name
  replicas: 1
  strategy:
    type: RollingUpdate
  revisionHistoryLimit: 10
  template:
    metadata:
      labels:        
        helm.sh/chart: kube-state-metrics-5.16.0
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: metrics
        app.kubernetes.io/part-of: kube-state-metrics
        app.kubernetes.io/name: kube-state-metrics
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.10.1"
        release: release-name
    spec:
      hostNetwork: false
      serviceAccountName: release-name-kube-state-metrics
      securityContext:
        fsGroup: 65534
        runAsGroup: 65534
        runAsNonRoot: true
        runAsUser: 65534
        seccompProfile:
          type: RuntimeDefault
      containers:
      - name: kube-state-metrics
        args:
        - --port=8080
        - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments
        imagePullPolicy: IfNotPresent
        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.10.1
        ports:
        - containerPort: 8080
          name: "http"
        livenessProbe:
          failureThreshold: 3
          httpGet:
            httpHeaders:
            path: /healthz
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        readinessProbe:
          failureThreshold: 3
          httpGet:
            httpHeaders:
            path: /
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 5
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 5
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
---
# Source: mimir/charts/mimir/charts/minio/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-minio
  labels:
    app: minio
    chart: minio-5.0.14
    release: release-name
    heritage: Helm
spec:
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 100%
      maxUnavailable: 0
  replicas: 1
  selector:
    matchLabels:
      app: minio
      release: release-name
  template:
    metadata:
      name: release-name-minio
      labels:
        app: minio
        release: release-name
      annotations:
        checksum/secrets: c8339c63fe17e834336f905a9b41c8c19ae107390219fc9b9ad71d3783e47601
        checksum/config: 1cd78d08302cd4659fe486040b24a41a7b9528e97c02fe2446ae70478df68cfd
    spec:
      securityContext:
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
        fsGroupChangePolicy: OnRootMismatch
      
      serviceAccountName: minio-sa
      containers:
        - name: minio
          image: "quay.io/minio/minio:RELEASE.2023-09-30T07-02-29Z"
          imagePullPolicy: IfNotPresent
          command:
            - "/bin/sh"
            - "-ce"
            - "/usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001"
          volumeMounts:
            - name: minio-user
              mountPath: "/tmp/credentials"
              readOnly: true
            - name: export
              mountPath: /export            
          ports:
            - name: http
              containerPort: 9000
            - name: http-console
              containerPort: 9001
          env:
            - name: MINIO_ROOT_USER
              valueFrom:
                secretKeyRef:
                  name: release-name-minio
                  key: rootUser
            - name: MINIO_ROOT_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: release-name-minio
                  key: rootPassword
            - name: MINIO_PROMETHEUS_AUTH_TYPE
              value: "public"
          resources:
            requests:
              cpu: 100m
              memory: 128Mi      
      volumes:
        - name: export
          persistentVolumeClaim:
            claimName: release-name-minio
        - name: minio-user
          secret:
            secretName: release-name-minio
---
# Source: mimir/charts/mimir/charts/rollout_operator/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-rollout-operator
  labels:
    helm.sh/chart: rollout-operator-0.10.0
    app.kubernetes.io/name: rollout-operator
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "v0.9.0"
    app.kubernetes.io/managed-by: Helm
spec:
  replicas: 1
  minReadySeconds: 10
  selector:
    matchLabels:
      app.kubernetes.io/name: rollout-operator
      app.kubernetes.io/instance: release-name
  strategy:
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
  template:
    metadata:
      labels:
        app.kubernetes.io/name: rollout-operator
        app.kubernetes.io/instance: release-name
    spec:
      serviceAccountName: release-name-rollout-operator
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: rollout-operator
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          image: "grafana/rollout-operator:v0.9.0"
          imagePullPolicy: IfNotPresent
          args:
          - -kubernetes.namespace=default
          ports:
            - name: http-metrics
              containerPort: 8001
              protocol: TCP
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 5
            timeoutSeconds: 1
          resources:
            limits:
              cpu: "1"
              memory: 200Mi
            requests:
              cpu: 100m
              memory: 100Mi
---
# Source: mimir/charts/mimir/templates/distributor/distributor-dep.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-mimir-distributor
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: distributor
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
  replicas: 2
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: distributor
  strategy:
    rollingUpdate:
      maxSurge: 15%
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: distributor
        app.kubernetes.io/part-of: memberlist
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      containers:
        - name: distributor
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=distributor"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
              subPath: 
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          resources:
            limits:
              cpu: 3.5
              memory: 5.7Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
            - name: "GOMAXPROCS"
              value: "8"
          envFrom:
      nodeSelector:
        {}
      affinity:
        {}
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: distributor
      tolerations:
        []
      terminationGracePeriodSeconds: 60
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: storage
          emptyDir: {}
        - name: active-queries
          emptyDir: {}
---
# Source: mimir/charts/mimir/templates/nginx/nginx-dep.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-mimir-nginx
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: nginx
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  replicas: 1
  strategy:
    rollingUpdate:
      maxSurge: 15%
      maxUnavailable: 0
    type: RollingUpdate
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: nginx
  template:
    metadata:
      annotations:
        checksum/config: 3207e30c8838f0e7fa2003ecc19522fc61ad4af5452ee2f1bf7bfe7312d9c21e
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: nginx
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      terminationGracePeriodSeconds: 30
      containers:
        - name: nginx
          image: docker.io/nginxinc/nginx-unprivileged:1.25-alpine
          imagePullPolicy: IfNotPresent
          ports:
            - name: http-metric
              containerPort: 8080
              protocol: TCP
          env:
          envFrom:
          readinessProbe:
            httpGet:
              path: /
              port: http-metric
            initialDelaySeconds: 15
            timeoutSeconds: 1
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          volumeMounts:
            - name: config
              mountPath: /etc/nginx
            - name: tmp
              mountPath: /tmp
            - name: docker-entrypoint-d-override
              mountPath: /docker-entrypoint.d
          resources:
            limits:
              cpu: 1.4
              memory: 731Mi
            requests:
              cpu: 20m
              memory: 10Mi
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: nginx
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-nginx
        - name: tmp
          emptyDir: {}
        - name: docker-entrypoint-d-override
          emptyDir: {}
---
# Source: mimir/charts/mimir/templates/overrides-exporter/overrides-exporter-dep.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    {}
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: overrides-exporter
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  name: release-name-mimir-overrides-exporter
  namespace: "default"
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: overrides-exporter
  strategy:
    rollingUpdate:
      maxSurge: 15%
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: overrides-exporter
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      containers:
        - name: overrides-exporter
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=overrides-exporter"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
              subPath: 
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          resources:
            limits:
              cpu: 100m
              memory: 128Mi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
      nodeSelector:
        {}
      affinity:
        {}
      
      tolerations:
        []
      terminationGracePeriodSeconds: 60
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: storage
          emptyDir: {}
        - name: active-queries
          emptyDir: {}
---
# Source: mimir/charts/mimir/templates/querier/querier-dep.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-mimir-querier
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: querier
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: querier
  strategy:
    rollingUpdate:
      maxSurge: 15%
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: querier
        app.kubernetes.io/part-of: memberlist
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      containers:
        - name: querier
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=querier"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
              subPath: 
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          resources:
            limits:
              cpu: 2.8
              memory: 5.6Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
            - name: "GOMAXPROCS"
              value: "5"
          envFrom:
      nodeSelector:
        {}
      affinity:
        {}
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: querier
      tolerations:
        []
      terminationGracePeriodSeconds: 180
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: storage
          emptyDir: {}
        - name: active-queries
          emptyDir: {}
---
# Source: mimir/charts/mimir/templates/query-frontend/query-frontend-dep.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-mimir-query-frontend
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-frontend
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  # If replicas is not number (when using values file it's float64, when using --set arg it's int64) and is false (i.e. null) don't set it
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: query-frontend
  strategy:
    rollingUpdate:
      maxSurge: 15%
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: query-frontend
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      containers:
        - name: query-frontend
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=query-frontend"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
          volumeMounts:
            - name: runtime-config
              mountPath: /var/mimir
            - name: config
              mountPath: /etc/mimir
            - name: storage
              mountPath: /data
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          resources:
            limits:
              cpu: 2.8
              memory: 2.8Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
      nodeSelector:
        {}
      affinity:
        {}
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: query-frontend
      tolerations:
        []
      terminationGracePeriodSeconds: 180
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: storage
          emptyDir: {}
        - name: active-queries
          emptyDir: {}
---
# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-dep.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-mimir-query-scheduler
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-scheduler
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
spec:
  replicas: 2
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: query-scheduler
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: query-scheduler
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      containers:
        - name: query-scheduler
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=query-scheduler"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
            - "-server.grpc.keepalive.max-connection-age=2562047h" # 100000 days, effectively infinity
            - "-server.grpc.keepalive.max-connection-age-grace=2562047h" # 100000 days, effectively infinity
          volumeMounts:
            - name: runtime-config
              mountPath: /var/mimir
            - name: config
              mountPath: /etc/mimir
            - name: storage
              mountPath: /data
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
      nodeSelector:
        {}
      affinity:
        {}
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: query-scheduler
      tolerations:
        []
      terminationGracePeriodSeconds: 180
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: storage
          emptyDir: {}
        - name: active-queries
          emptyDir: {}
---
# Source: mimir/charts/mimir/templates/ruler/ruler-dep.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: release-name-mimir-ruler
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ruler
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: ruler
  strategy:
    rollingUpdate:
      maxSurge: 50%
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: ruler
        app.kubernetes.io/part-of: memberlist
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      containers:
        - name: ruler
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=ruler"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
              subPath: 
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          resources:
            limits:
              cpu: 1.4
              memory: 2.8Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
      nodeSelector:
        {}
      affinity:
        {}
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: ruler
      tolerations:
        []
      terminationGracePeriodSeconds: 180
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: storage
          emptyDir: {}
        - name: active-queries
          emptyDir: {}
---
# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-alertmanager
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: alertmanager
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  replicas: 2
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: alertmanager
  updateStrategy:
    type: RollingUpdate
  serviceName: release-name-mimir-alertmanager
  volumeClaimTemplates:
    - metadata:
        name: storage
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: "1Gi"
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: alertmanager
        app.kubernetes.io/part-of: memberlist
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
        checksum/alertmanager-fallback-config: d2d10e73bd2feee8e9f9250f8ef6b276de6d4b2eeb9127a16e77eb0f5db7718f
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        {}
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: alertmanager
      tolerations:
        []
      terminationGracePeriodSeconds: 60
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: tmp
          emptyDir: {}
        - name: active-queries
          emptyDir: {}
        - name: alertmanager-fallback-config
          configMap:
            name: release-name-mimir-alertmanager-fallback-config
      containers:
        - name: alertmanager
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=alertmanager"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
            - name: alertmanager-fallback-config
              mountPath: /configs/
            - name: tmp
              mountPath: /tmp
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          resources:
            limits:
              cpu: 1.4
              memory: 1.4Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
---
# Source: mimir/charts/mimir/templates/compactor/compactor-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-compactor
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: compactor
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  podManagementPolicy: OrderedReady
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: compactor
  updateStrategy:
    type: RollingUpdate
  serviceName: release-name-mimir-compactor
  volumeClaimTemplates:
    - metadata:
        name: storage
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: "20Gi"
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: compactor
        app.kubernetes.io/part-of: memberlist
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        {}
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: compactor
      tolerations:
        []
      terminationGracePeriodSeconds: 240
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: active-queries
          emptyDir: {}
      containers:
        - name: compactor
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=compactor"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 60
          resources:
            limits:
              cpu: 1.4
              memory: 2.1Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
---
# Source: mimir/charts/mimir/templates/index-cache/index-cache-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-index-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: memcached
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  podManagementPolicy: Parallel
  replicas: 3
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: index-cache
  updateStrategy:
    type: RollingUpdate
  serviceName: release-name-mimir-index-cache

  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: index-cache
      annotations:

    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        {}
      
      tolerations:
        []
      terminationGracePeriodSeconds: 60
      containers:
        - name: memcached
          image: memcached:1.6.22-alpine
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              memory: 2458Mi
            requests:
              cpu: 500m
              memory: 2458Mi
          ports:
            - containerPort: 11211
              name: client
          args:
            - -m 2048
            - --extended=modern,track_sizes
            - -I 5m
            - -c 16384
            - -v
            - -u 11211
          env:
          envFrom:
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
        - name: exporter
          image: prom/memcached-exporter:v0.14.2
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 9150
              name: http-metrics
          args:
            - "--memcached.address=localhost:11211"
            - "--web.listen-address=0.0.0.0:9150"
          resources:
            limits: {}
            requests: {}
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
---
# Source: mimir/charts/mimir/templates/ingester/ingester-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-ingester-zone-a
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "ingester-zone-a"
    rollout-group: ingester
    zone: zone-a
  annotations:
    rollout-max-unavailable: "50"
  namespace: "default"
spec:
  podManagementPolicy: Parallel
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: ingester
      rollout-group: ingester
      zone: zone-a
  updateStrategy:
    type: OnDelete
  serviceName: release-name-mimir-ingester-headless
  volumeClaimTemplates:
    - metadata:
        name: storage
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: "50Gi"
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: ingester
        app.kubernetes.io/part-of: memberlist
        name: "ingester-zone-a"
        rollout-group: ingester
        zone: zone-a
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: rollout-group
                operator: In
                values:
                - ingester
              - key: zone
                operator: NotIn
                values:
                - zone-a
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: ingester
      tolerations:
        []
      terminationGracePeriodSeconds: 240
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: active-queries
          emptyDir: {}
      containers:
        - name: ingester
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=ingester"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
            - "-ingester.ring.instance-availability-zone=zone-a"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 60
          resources:
            limits:
              cpu: 5
              memory: 12Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
---
# Source: mimir/charts/mimir/templates/ingester/ingester-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-ingester-zone-b
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "ingester-zone-b"
    rollout-group: ingester
    zone: zone-b
  annotations:
    rollout-max-unavailable: "50"
  namespace: "default"
spec:
  podManagementPolicy: Parallel
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: ingester
      rollout-group: ingester
      zone: zone-b
  updateStrategy:
    type: OnDelete
  serviceName: release-name-mimir-ingester-headless
  volumeClaimTemplates:
    - metadata:
        name: storage
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: "50Gi"
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: ingester
        app.kubernetes.io/part-of: memberlist
        name: "ingester-zone-b"
        rollout-group: ingester
        zone: zone-b
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: rollout-group
                operator: In
                values:
                - ingester
              - key: zone
                operator: NotIn
                values:
                - zone-b
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: ingester
      tolerations:
        []
      terminationGracePeriodSeconds: 240
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: active-queries
          emptyDir: {}
      containers:
        - name: ingester
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=ingester"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
            - "-ingester.ring.instance-availability-zone=zone-b"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 60
          resources:
            limits:
              cpu: 5
              memory: 12Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
---
# Source: mimir/charts/mimir/templates/ingester/ingester-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-ingester-zone-c
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "ingester-zone-c"
    rollout-group: ingester
    zone: zone-c
  annotations:
    rollout-max-unavailable: "50"
  namespace: "default"
spec:
  podManagementPolicy: Parallel
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: ingester
      rollout-group: ingester
      zone: zone-c
  updateStrategy:
    type: OnDelete
  serviceName: release-name-mimir-ingester-headless
  volumeClaimTemplates:
    - metadata:
        name: storage
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: "50Gi"
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: ingester
        app.kubernetes.io/part-of: memberlist
        name: "ingester-zone-c"
        rollout-group: ingester
        zone: zone-c
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: rollout-group
                operator: In
                values:
                - ingester
              - key: zone
                operator: NotIn
                values:
                - zone-c
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: ingester
      tolerations:
        []
      terminationGracePeriodSeconds: 240
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: active-queries
          emptyDir: {}
      containers:
        - name: ingester
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=ingester"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
            - "-ingester.ring.instance-availability-zone=zone-c"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 60
          resources:
            limits:
              cpu: 5
              memory: 12Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
          envFrom:
---
# Source: mimir/charts/mimir/templates/metadata-cache/metadata-cache-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-metadata-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: memcached
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  podManagementPolicy: Parallel
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: metadata-cache
  updateStrategy:
    type: RollingUpdate
  serviceName: release-name-mimir-metadata-cache

  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: metadata-cache
      annotations:

    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        {}
      
      tolerations:
        []
      terminationGracePeriodSeconds: 60
      containers:
        - name: memcached
          image: memcached:1.6.22-alpine
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              memory: 614Mi
            requests:
              cpu: 500m
              memory: 614Mi
          ports:
            - containerPort: 11211
              name: client
          args:
            - -m 512
            - --extended=modern,track_sizes
            - -I 1m
            - -c 16384
            - -v
            - -u 11211
          env:
          envFrom:
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
        - name: exporter
          image: prom/memcached-exporter:v0.14.2
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 9150
              name: http-metrics
          args:
            - "--memcached.address=localhost:11211"
            - "--web.listen-address=0.0.0.0:9150"
          resources:
            limits: {}
            requests: {}
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
---
# Source: mimir/charts/mimir/templates/results-cache/results-cache-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-results-cache
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: memcached
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    {}
  namespace: "default"
spec:
  podManagementPolicy: Parallel
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: results-cache
  updateStrategy:
    type: RollingUpdate
  serviceName: release-name-mimir-results-cache

  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: results-cache
      annotations:

    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        {}
      
      tolerations:
        []
      terminationGracePeriodSeconds: 60
      containers:
        - name: memcached
          image: memcached:1.6.22-alpine
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              memory: 614Mi
            requests:
              cpu: 500m
              memory: 614Mi
          ports:
            - containerPort: 11211
              name: client
          args:
            - -m 512
            - --extended=modern,track_sizes
            - -I 5m
            - -c 16384
            - -v
            - -u 11211
          env:
          envFrom:
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
        - name: exporter
          image: prom/memcached-exporter:v0.14.2
          imagePullPolicy: IfNotPresent
          ports:
            - containerPort: 9150
              name: http-metrics
          args:
            - "--memcached.address=localhost:11211"
            - "--web.listen-address=0.0.0.0:9150"
          resources:
            limits: {}
            requests: {}
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-store-gateway-zone-a
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "store-gateway-zone-a"
    rollout-group: store-gateway
    zone: zone-a
  annotations:
    rollout-max-unavailable: "50"
  namespace: "default"
spec:
  podManagementPolicy: OrderedReady
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: store-gateway
      rollout-group: store-gateway
      zone: zone-a
  updateStrategy:
    type: OnDelete
  serviceName: release-name-mimir-store-gateway-headless
  volumeClaimTemplates:
    - metadata:
        name: storage
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: "10Gi"
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: store-gateway
        app.kubernetes.io/part-of: memberlist
        name: "store-gateway-zone-a"
        rollout-group: store-gateway
        zone: zone-a
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: rollout-group
                operator: In
                values:
                - store-gateway
              - key: zone
                operator: NotIn
                values:
                - zone-a
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: store-gateway
      tolerations:
        []
      terminationGracePeriodSeconds: 240
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: active-queries
          emptyDir: {}
      containers:
        - name: store-gateway
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=store-gateway"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
            - "-store-gateway.sharding-ring.instance-availability-zone=zone-a"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 60
          resources:
            limits:
              cpu: 1.4
              memory: 2.1Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
            - name: "GOMAXPROCS"
              value: "5"
            - name: "GOMEMLIMIT"
              value: "10485760"
          envFrom:
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-store-gateway-zone-b
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "store-gateway-zone-b"
    rollout-group: store-gateway
    zone: zone-b
  annotations:
    rollout-max-unavailable: "50"
  namespace: "default"
spec:
  podManagementPolicy: OrderedReady
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: store-gateway
      rollout-group: store-gateway
      zone: zone-b
  updateStrategy:
    type: OnDelete
  serviceName: release-name-mimir-store-gateway-headless
  volumeClaimTemplates:
    - metadata:
        name: storage
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: "10Gi"
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: store-gateway
        app.kubernetes.io/part-of: memberlist
        name: "store-gateway-zone-b"
        rollout-group: store-gateway
        zone: zone-b
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: rollout-group
                operator: In
                values:
                - store-gateway
              - key: zone
                operator: NotIn
                values:
                - zone-b
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: store-gateway
      tolerations:
        []
      terminationGracePeriodSeconds: 240
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: active-queries
          emptyDir: {}
      containers:
        - name: store-gateway
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=store-gateway"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
            - "-store-gateway.sharding-ring.instance-availability-zone=zone-b"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 60
          resources:
            limits:
              cpu: 1.4
              memory: 2.1Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
            - name: "GOMAXPROCS"
              value: "5"
            - name: "GOMEMLIMIT"
              value: "10485760"
          envFrom:
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: release-name-mimir-store-gateway-zone-c
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
    name: "store-gateway-zone-c"
    rollout-group: store-gateway
    zone: zone-c
  annotations:
    rollout-max-unavailable: "50"
  namespace: "default"
spec:
  podManagementPolicy: OrderedReady
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: store-gateway
      rollout-group: store-gateway
      zone: zone-c
  updateStrategy:
    type: OnDelete
  serviceName: release-name-mimir-store-gateway-headless
  volumeClaimTemplates:
    - metadata:
        name: storage
      spec:
        accessModes:
          - ReadWriteOnce
        resources:
          requests:
            storage: "10Gi"
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: store-gateway
        app.kubernetes.io/part-of: memberlist
        name: "store-gateway-zone-c"
        rollout-group: store-gateway
        zone: zone-c
      annotations:
        checksum/config: 3bbab9faeefd65a3aba9210861962b6195215b5bfc2e0b73737c3a14be64a1f0
      namespace: "default"
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      nodeSelector:
        {}
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: rollout-group
                operator: In
                values:
                - store-gateway
              - key: zone
                operator: NotIn
                values:
                - zone-c
            topologyKey: kubernetes.io/hostname
      topologySpreadConstraints:
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: ScheduleAnyway
        labelSelector:
          matchLabels:
            app.kubernetes.io/name: mimir
            app.kubernetes.io/instance: release-name
            app.kubernetes.io/component: store-gateway
      tolerations:
        []
      terminationGracePeriodSeconds: 240
      volumes:
        - name: config
          configMap:
            name: release-name-mimir-config
            items:
              - key: "mimir.yaml"
                path: "mimir.yaml"
        - name: runtime-config
          configMap:
            name: release-name-mimir-runtime
        - name: active-queries
          emptyDir: {}
      containers:
        - name: store-gateway
          image: "grafana/mimir:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-target=store-gateway"
            - "-config.expand-env=true"
            - "-config.file=/etc/mimir/mimir.yaml"
            - "-store-gateway.sharding-ring.instance-availability-zone=zone-c"
          volumeMounts:
            - name: config
              mountPath: /etc/mimir
            - name: runtime-config
              mountPath: /var/mimir
            - name: storage
              mountPath: "/data"
            - name: active-queries
              mountPath: /active-query-tracker
          ports:
            - name: http-metrics
              containerPort: 8080
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            null
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 60
          resources:
            limits:
              cpu: 1.4
              memory: 2.1Gi
            requests:
              cpu: 20m
              memory: 10Mi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          env:
            - name: "GOMAXPROCS"
              value: "5"
            - name: "GOMEMLIMIT"
              value: "10485760"
          envFrom:
---
# Source: mimir/charts/mimir/templates/minio/create-bucket-job.yaml
# Minio provides post-install hook to create bucket
# however the hook won't be executed if helm install is run
# with --wait flag. Hence this job is a workaround for that.
# See https://github.com/grafana/mimir/issues/2464
apiVersion: batch/v1
kind: Job
metadata:
  name: release-name-mimir-make-minio-buckets-5.0.14
  namespace: "default"
  labels:
    app: mimir-make-bucket-job
    chart: mimir-5.2.1
    release: release-name
    heritage: Helm
spec:
  template:
    metadata:
      labels:
        app: mimir-job
        release: release-name
    spec:
      restartPolicy: OnFailure      
      volumes:
        - name: minio-configuration
          projected:
            sources:
            - configMap:
                name: release-name-minio
            - secret:
                name: release-name-minio
      containers:
      - name: minio-mc
        image: "quay.io/minio/mc:RELEASE.2023-09-29T16-41-22Z"
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "/config/initialize"]
        env:
          - name: MINIO_ENDPOINT
            value: release-name-minio
          - name: MINIO_PORT
            value: "9000"
        volumeMounts:
          - name: minio-configuration
            mountPath: /config
        resources:
          requests:
            memory: 128Mi
---
# Source: mimir/charts/mimir/templates/metamonitoring/grafana-agent.yaml
apiVersion: monitoring.grafana.com/v1alpha1
kind: GrafanaAgent
metadata:
  name: release-name-mimir-meta-monitoring
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: meta-monitoring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  serviceAccountName: release-name-mimir
  containers:
    # The container specs here are merged with the ones in the operator using a strategic merge patch.
    - name: config-reloader
    - name: grafana-agent

  logs:
    instanceSelector:
      matchLabels:
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/component: meta-monitoring
    # cluster label for logs is added in the LogsInstance
  metrics:
    scrapeInterval: 60s
    instanceSelector:
      matchLabels:
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/component: meta-monitoring
    externalLabels:
      cluster: release-name
---
# Source: mimir/charts/mimir/templates/metamonitoring/logs-instance.yaml
apiVersion: monitoring.grafana.com/v1alpha1
kind: LogsInstance
metadata:
  name: release-name-mimir-meta-monitoring
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: meta-monitoring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  clients:
      

  # Supply an empty namespace selector to look in all namespaces. Remove
  # this to only look in the same namespace as the LogsInstance CR
  podLogsNamespaceSelector: {}

  podLogsSelector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
---
# Source: mimir/charts/mimir/templates/metamonitoring/metrics-instance.yaml
apiVersion: monitoring.grafana.com/v1alpha1
kind: MetricsInstance
metadata:
  name: release-name-mimir-meta-monitoring
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: meta-monitoring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  remoteWrite:
      - url: http://{{ if ne .Release.Name "mimir" }}{{ .Release.Name }}-{{ end }}mimir-nginx.{{ .Release.Namespace }}/prometheus
        basicAuth:

  # Supply an empty namespace selector to look in all namespaces. Remove
  # this to only look in the same namespace as the MetricsInstance CR
  serviceMonitorNamespaceSelector: {}

  serviceMonitorSelector:
    # Scrape ServiceMonitors from all components
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
---
# Source: mimir/charts/mimir/templates/metamonitoring/pod-logs.yaml
apiVersion: monitoring.grafana.com/v1alpha1
kind: PodLogs
metadata:
  name: release-name-mimir-meta-monitoring
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: meta-monitoring
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  pipelineStages:
    - cri: { }
  relabelings:
    - action: replace # For consistency with metrics
      replacement: $1
      separator: /
      sourceLabels:
        - __meta_kubernetes_namespace
        - __meta_kubernetes_pod_container_name
      targetLabel: job
    - action: replace # Necessary for slow queries dashboard
      sourceLabels:
        - __meta_kubernetes_pod_container_name
      targetLabel: name
    - targetLabel: cluster
      replacement: release-name

  namespaceSelector:
    matchNames:
      - "default"

  selector:
    matchLabels:
      # Scrape logs from all components
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/alertmanager.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-alertmanager.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: alertmanager.rules
    rules:
    - alert: AlertmanagerFailedReload
      annotations:
        description: Configuration has failed to load for {{ $labels.namespace }}/{{ $labels.pod}}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerfailedreload
        summary: Reloading an Alertmanager configuration has failed.
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        max_over_time(alertmanager_config_last_reload_successful{job="release-name-kps-alertmanager",namespace="default"}[5m]) == 0
      for: 10m
      labels:
        severity: critical
    - alert: AlertmanagerMembersInconsistent
      annotations:
        description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} has only found {{ $value }} members of the {{$labels.job}} cluster.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagermembersinconsistent
        summary: A member of an Alertmanager cluster has not found all other cluster members.
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
          max_over_time(alertmanager_cluster_members{job="release-name-kps-alertmanager",namespace="default"}[5m])
        < on (namespace,service,cluster) group_left
          count by (namespace,service,cluster) (max_over_time(alertmanager_cluster_members{job="release-name-kps-alertmanager",namespace="default"}[5m]))
      for: 15m
      labels:
        severity: critical
    - alert: AlertmanagerFailedToSendAlerts
      annotations:
        description: Alertmanager {{ $labels.namespace }}/{{ $labels.pod}} failed to send {{ $value | humanizePercentage }} of notifications to {{ $labels.integration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerfailedtosendalerts
        summary: An Alertmanager instance failed to send notifications.
      expr: |-
        (
          rate(alertmanager_notifications_failed_total{job="release-name-kps-alertmanager",namespace="default"}[5m])
        /
          ignoring (reason) group_left rate(alertmanager_notifications_total{job="release-name-kps-alertmanager",namespace="default"}[5m])
        )
        > 0.01
      for: 5m
      labels:
        severity: warning
    - alert: AlertmanagerClusterFailedToSendAlerts
      annotations:
        description: The minimum notification failure rate to {{ $labels.integration }} sent from any instance in the {{$labels.job}} cluster is {{ $value | humanizePercentage }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterfailedtosendalerts
        summary: All Alertmanager instances in a cluster failed to send notifications to a critical integration.
      expr: |-
        min by (namespace,service, integration) (
          rate(alertmanager_notifications_failed_total{job="release-name-kps-alertmanager",namespace="default", integration=~`.*`}[5m])
        /
          ignoring (reason) group_left rate(alertmanager_notifications_total{job="release-name-kps-alertmanager",namespace="default", integration=~`.*`}[5m])
        )
        > 0.01
      for: 5m
      labels:
        severity: critical
    - alert: AlertmanagerClusterFailedToSendAlerts
      annotations:
        description: The minimum notification failure rate to {{ $labels.integration }} sent from any instance in the {{$labels.job}} cluster is {{ $value | humanizePercentage }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterfailedtosendalerts
        summary: All Alertmanager instances in a cluster failed to send notifications to a non-critical integration.
      expr: |-
        min by (namespace,service, integration) (
          rate(alertmanager_notifications_failed_total{job="release-name-kps-alertmanager",namespace="default", integration!~`.*`}[5m])
        /
          ignoring (reason) group_left rate(alertmanager_notifications_total{job="release-name-kps-alertmanager",namespace="default", integration!~`.*`}[5m])
        )
        > 0.01
      for: 5m
      labels:
        severity: warning
    - alert: AlertmanagerConfigInconsistent
      annotations:
        description: Alertmanager instances within the {{$labels.job}} cluster have different configurations.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerconfiginconsistent
        summary: Alertmanager instances within the same cluster have different configurations.
      expr: |-
        count by (namespace,service,cluster) (
          count_values by (namespace,service,cluster) ("config_hash", alertmanager_config_hash{job="release-name-kps-alertmanager",namespace="default"})
        )
        != 1
      for: 20m
      labels:
        severity: critical
    - alert: AlertmanagerClusterDown
      annotations:
        description: '{{ $value | humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have been up for less than half of the last 5m.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclusterdown
        summary: Half or more of the Alertmanager instances within the same cluster are down.
      expr: |-
        (
          count by (namespace,service,cluster) (
            avg_over_time(up{job="release-name-kps-alertmanager",namespace="default"}[5m]) < 0.5
          )
        /
          count by (namespace,service,cluster) (
            up{job="release-name-kps-alertmanager",namespace="default"}
          )
        )
        >= 0.5
      for: 5m
      labels:
        severity: critical
    - alert: AlertmanagerClusterCrashlooping
      annotations:
        description: '{{ $value | humanizePercentage }} of Alertmanager instances within the {{$labels.job}} cluster have restarted at least 5 times in the last 10m.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/alertmanager/alertmanagerclustercrashlooping
        summary: Half or more of the Alertmanager instances within the same cluster are crashlooping.
      expr: |-
        (
          count by (namespace,service,cluster) (
            changes(process_start_time_seconds{job="release-name-kps-alertmanager",namespace="default"}[10m]) > 4
          )
        /
          count by (namespace,service,cluster) (
            up{job="release-name-kps-alertmanager",namespace="default"}
          )
        )
        >= 0.5
      for: 5m
      labels:
        severity: critical
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/config-reloaders.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-config-reloaders
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: config-reloaders
    rules:
    - alert: ConfigReloaderSidecarErrors
      annotations:
        description: 'Errors encountered while the {{$labels.pod}} config-reloader sidecar attempts to sync config in {{$labels.namespace}} namespace.

          As a result, configuration for service running in {{$labels.pod}} may be stale and cannot be updated anymore.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/configreloadersidecarerrors
        summary: config-reloader sidecar has not had a successful reload for 10m
      expr: max_over_time(reloader_last_reload_successful{namespace=~".+"}[5m]) == 0
      for: 10m
      labels:
        severity: warning
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/etcd.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-etcd
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: etcd
    rules:
    - alert: etcdMembersDown
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": members are down ({{ $value }}).'
        summary: etcd cluster members are down.
      expr: |-
        max without (endpoint) (
          sum without (instance) (up{job=~".*etcd.*"} == bool 0)
        or
          count without (To) (
            sum without (instance) (rate(etcd_network_peer_sent_failures_total{job=~".*etcd.*"}[120s])) > 0.01
          )
        )
        > 0
      for: 10m
      labels:
        severity: critical
    - alert: etcdInsufficientMembers
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value }}).'
        summary: etcd cluster has insufficient number of members.
      expr: sum(up{job=~".*etcd.*"} == bool 1) without (instance) < ((count(up{job=~".*etcd.*"}) without (instance) + 1) / 2)
      for: 3m
      labels:
        severity: critical
    - alert: etcdNoLeader
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": member {{ $labels.instance }} has no leader.'
        summary: etcd cluster has no leader.
      expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
      for: 1m
      labels:
        severity: critical
    - alert: etcdHighNumberOfLeaderChanges
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated.'
        summary: etcd cluster has high number of leader changes.
      expr: increase((max without (instance) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0*absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 4
      for: 5m
      labels:
        severity: warning
    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster has high number of failed grpc requests.
      expr: |-
        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
          /
        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
          > 1
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighNumberOfFailedGRPCRequests
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster has high number of failed grpc requests.
      expr: |-
        100 * sum(rate(grpc_server_handled_total{job=~".*etcd.*", grpc_code=~"Unknown|FailedPrecondition|ResourceExhausted|Internal|Unavailable|DataLoss|DeadlineExceeded"}[5m])) without (grpc_type, grpc_code)
          /
        sum(rate(grpc_server_handled_total{job=~".*etcd.*"}[5m])) without (grpc_type, grpc_code)
          > 5
      for: 5m
      labels:
        severity: critical
    - alert: etcdGRPCRequestsSlow
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile of gRPC requests is {{ $value }}s on etcd instance {{ $labels.instance }} for {{ $labels.grpc_method }} method.'
        summary: etcd grpc requests are slow
      expr: |-
        histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_method!="Defragment", grpc_type="unary"}[5m])) without(grpc_type))
        > 0.15
      for: 10m
      labels:
        severity: critical
    - alert: etcdMemberCommunicationSlow
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": member communication with {{ $labels.To }} is taking {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster member communication is slow.
      expr: |-
        histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.15
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighNumberOfFailedProposals
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": {{ $value }} proposal failures within the last 30 minutes on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster has high number of proposal failures.
      expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
      for: 15m
      labels:
        severity: warning
    - alert: etcdHighFsyncDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: |-
        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.5
      for: 10m
      labels:
        severity: warning
    - alert: etcdHighFsyncDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile fsync durations are too high.
      expr: |-
        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 1
      for: 10m
      labels:
        severity: critical
    - alert: etcdHighCommitDurations
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.'
        summary: etcd cluster 99th percentile commit durations are too high.
      expr: |-
        histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
        > 0.25
      for: 10m
      labels:
        severity: warning
    - alert: etcdDatabaseQuotaLowSpace
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": database size exceeds the defined quota on etcd instance {{ $labels.instance }}, please defrag or increase the quota as the writes to etcd will be disabled when it is full.'
        summary: etcd cluster database is running full.
      expr: (last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_server_quota_backend_bytes{job=~".*etcd.*"}[5m]))*100 > 95
      for: 10m
      labels:
        severity: critical
    - alert: etcdExcessiveDatabaseGrowth
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.'
        summary: etcd cluster database growing very fast.
      expr: predict_linear(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[4h], 4*60*60) > etcd_server_quota_backend_bytes{job=~".*etcd.*"}
      for: 10m
      labels:
        severity: warning
    - alert: etcdDatabaseHighFragmentationRatio
      annotations:
        description: 'etcd cluster "{{ $labels.job }}": database size in use on instance {{ $labels.instance }} is {{ $value | humanizePercentage }} of the actual allocated disk space, please run defragmentation (e.g. etcdctl defrag) to retrieve the unused fragmented disk space.'
        runbook_url: https://etcd.io/docs/v3.5/op-guide/maintenance/#defragmentation
        summary: etcd database size in use is less than 50% of the actual allocated storage.
      expr: (last_over_time(etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"}[5m]) / last_over_time(etcd_mvcc_db_total_size_in_bytes{job=~".*etcd.*"}[5m])) < 0.5 and etcd_mvcc_db_total_size_in_use_in_bytes{job=~".*etcd.*"} > 104857600
      for: 10m
      labels:
        severity: warning
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/general.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-general.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: general.rules
    rules:
    - alert: TargetDown
      annotations:
        description: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service }} targets in {{ $labels.namespace }} namespace are down.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/targetdown
        summary: One or more targets are unreachable.
      expr: 100 * (count(up == 0) BY (cluster, job, namespace, service) / count(up) BY (cluster, job, namespace, service)) > 10
      for: 10m
      labels:
        severity: warning
    - alert: Watchdog
      annotations:
        description: 'This is an alert meant to ensure that the entire alerting pipeline is functional.

          This alert is always firing, therefore it should always be firing in Alertmanager

          and always fire against a receiver. There are integrations with various notification

          mechanisms that send a notification when this alert is not firing. For example the

          "DeadMansSnitch" integration in PagerDuty.

          '
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/watchdog
        summary: An alert that should always be firing to certify that Alertmanager is working properly.
      expr: vector(1)
      labels:
        severity: none
    - alert: InfoInhibitor
      annotations:
        description: 'This is an alert that is used to inhibit info alerts.

          By themselves, the info-level alerts are sometimes very noisy, but they are relevant when combined with

          other alerts.

          This alert fires whenever there''s a severity="info" alert, and stops firing when another alert with a

          severity of ''warning'' or ''critical'' starts firing on the same namespace.

          This alert should be routed to a null receiver and configured to inhibit alerts with severity="info".

          '
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/infoinhibitor
        summary: Info-level alert inhibition.
      expr: ALERTS{severity = "info"} == 1 unless on (namespace) ALERTS{alertname != "InfoInhibitor", severity =~ "warning|critical", alertstate="firing"} == 1
      labels:
        severity: none
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_cpu_usage_seconds_total.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-k8s.rules.container-cpu-usage-seconds-total
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: k8s.rules.container_cpu_usage_seconds_total
    rules:
    - expr: |-
        sum by (cluster, namespace, pod, container) (
          irate(container_cpu_usage_seconds_total{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}[5m])
        ) * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (
          1, max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
        )
      record: node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_memory_cache.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-k8s.rules.container-memory-cache
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: k8s.rules.container_memory_cache
    rules:
    - expr: |-
        container_memory_cache{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
        * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1,
          max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
        )
      record: node_namespace_pod_container:container_memory_cache
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_memory_rss.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-k8s.rules.container-memory-rss
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: k8s.rules.container_memory_rss
    rules:
    - expr: |-
        container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
        * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1,
          max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
        )
      record: node_namespace_pod_container:container_memory_rss
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_memory_swap.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-k8s.rules.container-memory-swap
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: k8s.rules.container_memory_swap
    rules:
    - expr: |-
        container_memory_swap{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
        * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1,
          max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
        )
      record: node_namespace_pod_container:container_memory_swap
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_memory_working_set_bytes.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-k8s.rules.container-memory-working-set-bytes
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: k8s.rules.container_memory_working_set_bytes
    rules:
    - expr: |-
        container_memory_working_set_bytes{job="kubelet", metrics_path="/metrics/cadvisor", image!=""}
        * on (cluster, namespace, pod) group_left(node) topk by (cluster, namespace, pod) (1,
          max by (cluster, namespace, pod, node) (kube_pod_info{node!=""})
        )
      record: node_namespace_pod_container:container_memory_working_set_bytes
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.container_resource.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-k8s.rules.container-resource
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: k8s.rules.container_resource
    rules:
    - expr: |-
        kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}  * on (namespace, pod, cluster)
        group_left() max by (namespace, pod, cluster) (
          (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
        )
      record: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests
    - expr: |-
        sum by (namespace, cluster) (
            sum by (namespace, pod, cluster) (
                max by (namespace, pod, container, cluster) (
                  kube_pod_container_resource_requests{resource="memory",job="kube-state-metrics"}
                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
                )
            )
        )
      record: namespace_memory:kube_pod_container_resource_requests:sum
    - expr: |-
        kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}  * on (namespace, pod, cluster)
        group_left() max by (namespace, pod, cluster) (
          (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
        )
      record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests
    - expr: |-
        sum by (namespace, cluster) (
            sum by (namespace, pod, cluster) (
                max by (namespace, pod, container, cluster) (
                  kube_pod_container_resource_requests{resource="cpu",job="kube-state-metrics"}
                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
                )
            )
        )
      record: namespace_cpu:kube_pod_container_resource_requests:sum
    - expr: |-
        kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}  * on (namespace, pod, cluster)
        group_left() max by (namespace, pod, cluster) (
          (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
        )
      record: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits
    - expr: |-
        sum by (namespace, cluster) (
            sum by (namespace, pod, cluster) (
                max by (namespace, pod, container, cluster) (
                  kube_pod_container_resource_limits{resource="memory",job="kube-state-metrics"}
                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
                )
            )
        )
      record: namespace_memory:kube_pod_container_resource_limits:sum
    - expr: |-
        kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}  * on (namespace, pod, cluster)
        group_left() max by (namespace, pod, cluster) (
         (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
         )
      record: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits
    - expr: |-
        sum by (namespace, cluster) (
            sum by (namespace, pod, cluster) (
                max by (namespace, pod, container, cluster) (
                  kube_pod_container_resource_limits{resource="cpu",job="kube-state-metrics"}
                ) * on (namespace, pod, cluster) group_left() max by (namespace, pod, cluster) (
                  kube_pod_status_phase{phase=~"Pending|Running"} == 1
                )
            )
        )
      record: namespace_cpu:kube_pod_container_resource_limits:sum
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/k8s.rules.pod_owner.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-k8s.rules.pod-owner
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: k8s.rules.pod_owner
    rules:
    - expr: |-
        max by (cluster, namespace, workload, pod) (
          label_replace(
            label_replace(
              kube_pod_owner{job="kube-state-metrics", owner_kind="ReplicaSet"},
              "replicaset", "$1", "owner_name", "(.*)"
            ) * on (replicaset, namespace) group_left(owner_name) topk by (replicaset, namespace) (
              1, max by (replicaset, namespace, owner_name) (
                kube_replicaset_owner{job="kube-state-metrics"}
              )
            ),
            "workload", "$1", "owner_name", "(.*)"
          )
        )
      labels:
        workload_type: deployment
      record: namespace_workload_pod:kube_pod_owner:relabel
    - expr: |-
        max by (cluster, namespace, workload, pod) (
          label_replace(
            kube_pod_owner{job="kube-state-metrics", owner_kind="DaemonSet"},
            "workload", "$1", "owner_name", "(.*)"
          )
        )
      labels:
        workload_type: daemonset
      record: namespace_workload_pod:kube_pod_owner:relabel
    - expr: |-
        max by (cluster, namespace, workload, pod) (
          label_replace(
            kube_pod_owner{job="kube-state-metrics", owner_kind="StatefulSet"},
            "workload", "$1", "owner_name", "(.*)"
          )
        )
      labels:
        workload_type: statefulset
      record: namespace_workload_pod:kube_pod_owner:relabel
    - expr: |-
        max by (cluster, namespace, workload, pod) (
          label_replace(
            kube_pod_owner{job="kube-state-metrics", owner_kind="Job"},
            "workload", "$1", "owner_name", "(.*)"
          )
        )
      labels:
        workload_type: job
      record: namespace_workload_pod:kube_pod_owner:relabel
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-apiserver-availability.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kube-apiserver-availability.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - interval: 3m
    name: kube-apiserver-availability.rules
    rules:
    - expr: avg_over_time(code_verb:apiserver_request_total:increase1h[30d]) * 24 * 30
      record: code_verb:apiserver_request_total:increase30d
    - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"LIST|GET"})
      labels:
        verb: read
      record: code:apiserver_request_total:increase30d
    - expr: sum by (cluster, code) (code_verb:apiserver_request_total:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
      labels:
        verb: write
      record: code:apiserver_request_total:increase30d
    - expr: sum by (cluster, verb, scope) (increase(apiserver_request_sli_duration_seconds_count{job="apiserver"}[1h]))
      record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h
    - expr: sum by (cluster, verb, scope) (avg_over_time(cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase1h[30d]) * 24 * 30)
      record: cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d
    - expr: sum by (cluster, verb, scope, le) (increase(apiserver_request_sli_duration_seconds_bucket[1h]))
      record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h
    - expr: sum by (cluster, verb, scope, le) (avg_over_time(cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase1h[30d]) * 24 * 30)
      record: cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d
    - expr: |-
        1 - (
          (
            # write too slow
            sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
            -
            sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
          ) +
          (
            # read too slow
            sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"})
            -
            (
              (
                sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
                or
                vector(0)
              )
              +
              sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
              +
              sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
            )
          ) +
          # errors
          sum by (cluster) (code:apiserver_request_total:increase30d{code=~"5.."} or vector(0))
        )
        /
        sum by (cluster) (code:apiserver_request_total:increase30d)
      labels:
        verb: all
      record: apiserver_request:availability30d
    - expr: |-
        1 - (
          sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"LIST|GET"})
          -
          (
            # too slow
            (
              sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope=~"resource|",le="1"})
              or
              vector(0)
            )
            +
            sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="namespace",le="5"})
            +
            sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"LIST|GET",scope="cluster",le="30"})
          )
          +
          # errors
          sum by (cluster) (code:apiserver_request_total:increase30d{verb="read",code=~"5.."} or vector(0))
        )
        /
        sum by (cluster) (code:apiserver_request_total:increase30d{verb="read"})
      labels:
        verb: read
      record: apiserver_request:availability30d
    - expr: |-
        1 - (
          (
            # too slow
            sum by (cluster) (cluster_verb_scope:apiserver_request_sli_duration_seconds_count:increase30d{verb=~"POST|PUT|PATCH|DELETE"})
            -
            sum by (cluster) (cluster_verb_scope_le:apiserver_request_sli_duration_seconds_bucket:increase30d{verb=~"POST|PUT|PATCH|DELETE",le="1"})
          )
          +
          # errors
          sum by (cluster) (code:apiserver_request_total:increase30d{verb="write",code=~"5.."} or vector(0))
        )
        /
        sum by (cluster) (code:apiserver_request_total:increase30d{verb="write"})
      labels:
        verb: write
      record: apiserver_request:availability30d
    - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
      labels:
        verb: read
      record: code_resource:apiserver_request_total:rate5m
    - expr: sum by (cluster,code,resource) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
      labels:
        verb: write
      record: code_resource:apiserver_request_total:rate5m
    - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"2.."}[1h]))
      record: code_verb:apiserver_request_total:increase1h
    - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"3.."}[1h]))
      record: code_verb:apiserver_request_total:increase1h
    - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"4.."}[1h]))
      record: code_verb:apiserver_request_total:increase1h
    - expr: sum by (cluster, code, verb) (increase(apiserver_request_total{job="apiserver",verb=~"LIST|GET|POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
      record: code_verb:apiserver_request_total:increase1h
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-apiserver-burnrate.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kube-apiserver-burnrate.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kube-apiserver-burnrate.rules
    rules:
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
            -
            (
              (
                sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1d]))
                or
                vector(0)
              )
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1d]))
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1d]))
            )
          )
          +
          # errors
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1d]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1d]))
      labels:
        verb: read
      record: apiserver_request:burnrate1d
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
            -
            (
              (
                sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[1h]))
                or
                vector(0)
              )
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[1h]))
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[1h]))
            )
          )
          +
          # errors
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[1h]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[1h]))
      labels:
        verb: read
      record: apiserver_request:burnrate1h
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
            -
            (
              (
                sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[2h]))
                or
                vector(0)
              )
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[2h]))
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[2h]))
            )
          )
          +
          # errors
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[2h]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[2h]))
      labels:
        verb: read
      record: apiserver_request:burnrate2h
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
            -
            (
              (
                sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[30m]))
                or
                vector(0)
              )
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[30m]))
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[30m]))
            )
          )
          +
          # errors
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[30m]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[30m]))
      labels:
        verb: read
      record: apiserver_request:burnrate30m
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
            -
            (
              (
                sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[3d]))
                or
                vector(0)
              )
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[3d]))
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[3d]))
            )
          )
          +
          # errors
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[3d]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[3d]))
      labels:
        verb: read
      record: apiserver_request:burnrate3d
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
            -
            (
              (
                sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[5m]))
                or
                vector(0)
              )
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[5m]))
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[5m]))
            )
          )
          +
          # errors
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[5m]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[5m]))
      labels:
        verb: read
      record: apiserver_request:burnrate5m
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
            -
            (
              (
                sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope=~"resource|",le="1"}[6h]))
                or
                vector(0)
              )
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="namespace",le="5"}[6h]))
              +
              sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward",scope="cluster",le="30"}[6h]))
            )
          )
          +
          # errors
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET",code=~"5.."}[6h]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"LIST|GET"}[6h]))
      labels:
        verb: read
      record: apiserver_request:burnrate6h
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1d]))
            -
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1d]))
          )
          +
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1d]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1d]))
      labels:
        verb: write
      record: apiserver_request:burnrate1d
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[1h]))
            -
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[1h]))
          )
          +
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[1h]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[1h]))
      labels:
        verb: write
      record: apiserver_request:burnrate1h
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[2h]))
            -
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[2h]))
          )
          +
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[2h]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[2h]))
      labels:
        verb: write
      record: apiserver_request:burnrate2h
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[30m]))
            -
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[30m]))
          )
          +
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[30m]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[30m]))
      labels:
        verb: write
      record: apiserver_request:burnrate30m
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[3d]))
            -
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[3d]))
          )
          +
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[3d]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[3d]))
      labels:
        verb: write
      record: apiserver_request:burnrate3d
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))
            -
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[5m]))
          )
          +
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[5m]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[5m]))
      labels:
        verb: write
      record: apiserver_request:burnrate5m
    - expr: |-
        (
          (
            # too slow
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_count{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[6h]))
            -
            sum by (cluster) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward",le="1"}[6h]))
          )
          +
          sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",code=~"5.."}[6h]))
        )
        /
        sum by (cluster) (rate(apiserver_request_total{job="apiserver",verb=~"POST|PUT|PATCH|DELETE"}[6h]))
      labels:
        verb: write
      record: apiserver_request:burnrate6h
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-apiserver-histogram.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kube-apiserver-histogram.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kube-apiserver-histogram.rules
    rules:
    - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"LIST|GET",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0
      labels:
        quantile: '0.99'
        verb: read
      record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.99, sum by (cluster, le, resource) (rate(apiserver_request_sli_duration_seconds_bucket{job="apiserver",verb=~"POST|PUT|PATCH|DELETE",subresource!~"proxy|attach|log|exec|portforward"}[5m]))) > 0
      labels:
        quantile: '0.99'
        verb: write
      record: cluster_quantile:apiserver_request_sli_duration_seconds:histogram_quantile
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-apiserver-slos.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kube-apiserver-slos
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kube-apiserver-slos
    rules:
    - alert: KubeAPIErrorBudgetBurn
      annotations:
        description: The API server is burning too much error budget.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
        summary: The API server is burning too much error budget.
      expr: |-
        sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
        and
        sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
      for: 2m
      labels:
        long: 1h
        severity: critical
        short: 5m
    - alert: KubeAPIErrorBudgetBurn
      annotations:
        description: The API server is burning too much error budget.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
        summary: The API server is burning too much error budget.
      expr: |-
        sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
        and
        sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
      for: 15m
      labels:
        long: 6h
        severity: critical
        short: 30m
    - alert: KubeAPIErrorBudgetBurn
      annotations:
        description: The API server is burning too much error budget.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
        summary: The API server is burning too much error budget.
      expr: |-
        sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
        and
        sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
      for: 1h
      labels:
        long: 1d
        severity: warning
        short: 2h
    - alert: KubeAPIErrorBudgetBurn
      annotations:
        description: The API server is burning too much error budget.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapierrorbudgetburn
        summary: The API server is burning too much error budget.
      expr: |-
        sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
        and
        sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
      for: 3h
      labels:
        long: 3d
        severity: warning
        short: 6h
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-prometheus-general.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kube-prometheus-general.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kube-prometheus-general.rules
    rules:
    - expr: count without(instance, pod, node) (up == 1)
      record: count:up1
    - expr: count without(instance, pod, node) (up == 0)
      record: count:up0
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-prometheus-node-recording.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kube-prometheus-node-recording.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kube-prometheus-node-recording.rules
    rules:
    - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[3m])) BY (instance)
      record: instance:node_cpu:rate:sum
    - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance)
      record: instance:node_network_receive_bytes:rate:sum
    - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)
      record: instance:node_network_transmit_bytes:rate:sum
    - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance)
      record: instance:node_cpu:ratio
    - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal"}[5m]))
      record: cluster:node_cpu:sum_rate5m
    - expr: cluster:node_cpu:sum_rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))
      record: cluster:node_cpu:ratio
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-scheduler.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kube-scheduler.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kube-scheduler.rules
    rules:
    - expr: histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.99'
      record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.99'
      record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.99'
      record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.9, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.9'
      record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.9, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.9'
      record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.9, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.9'
      record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.5, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.5'
      record: cluster_quantile:scheduler_e2e_scheduling_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.5, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.5'
      record: cluster_quantile:scheduler_scheduling_algorithm_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.5, sum(rate(scheduler_binding_duration_seconds_bucket{job="kube-scheduler"}[5m])) without(instance, pod))
      labels:
        quantile: '0.5'
      record: cluster_quantile:scheduler_binding_duration_seconds:histogram_quantile
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kube-state-metrics.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kube-state-metrics
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kube-state-metrics
    rules:
    - alert: KubeStateMetricsListErrors
      annotations:
        description: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricslisterrors
        summary: kube-state-metrics is experiencing errors in list operations.
      expr: |-
        (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
          /
        sum(rate(kube_state_metrics_list_total{job="kube-state-metrics"}[5m])) by (cluster))
        > 0.01
      for: 15m
      labels:
        severity: critical
    - alert: KubeStateMetricsWatchErrors
      annotations:
        description: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricswatcherrors
        summary: kube-state-metrics is experiencing errors in watch operations.
      expr: |-
        (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) by (cluster)
          /
        sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics"}[5m])) by (cluster))
        > 0.01
      for: 15m
      labels:
        severity: critical
    - alert: KubeStateMetricsShardingMismatch
      annotations:
        description: kube-state-metrics pods are running with different --total-shards configuration, some Kubernetes objects may be exposed multiple times or not exposed at all.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardingmismatch
        summary: kube-state-metrics sharding is misconfigured.
      expr: stdvar (kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) != 0
      for: 15m
      labels:
        severity: critical
    - alert: KubeStateMetricsShardsMissing
      annotations:
        description: kube-state-metrics shards are missing, some Kubernetes objects are not being exposed.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kube-state-metrics/kubestatemetricsshardsmissing
        summary: kube-state-metrics shards are missing.
      expr: |-
        2^max(kube_state_metrics_total_shards{job="kube-state-metrics"}) by (cluster) - 1
          -
        sum( 2 ^ max by (cluster, shard_ordinal) (kube_state_metrics_shard_ordinal{job="kube-state-metrics"}) ) by (cluster)
        != 0
      for: 15m
      labels:
        severity: critical
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubelet.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubelet.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubelet.rules
    rules:
    - expr: histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) * on (cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
      labels:
        quantile: '0.99'
      record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) * on (cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
      labels:
        quantile: '0.9'
      record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
    - expr: histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le) * on (cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"})
      labels:
        quantile: '0.5'
      record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-apps.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubernetes-apps
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubernetes-apps
    rules:
    - alert: KubePodCrashLooping
      annotations:
        description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is in waiting state (reason: "CrashLoopBackOff").'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodcrashlooping
        summary: Pod is crash looping.
      expr: max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", job="kube-state-metrics", namespace=~".*"}[5m]) >= 1
      for: 15m
      labels:
        severity: warning
    - alert: KubePodNotReady
      annotations:
        description: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepodnotready
        summary: Pod has been in a non-ready state for more than 15 minutes.
      expr: |-
        sum by (namespace, pod, cluster) (
          max by (namespace, pod, cluster) (
            kube_pod_status_phase{job="kube-state-metrics", namespace=~".*", phase=~"Pending|Unknown|Failed"}
          ) * on (namespace, pod, cluster) group_left(owner_kind) topk by (namespace, pod, cluster) (
            1, max by (namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"})
          )
        ) > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeDeploymentGenerationMismatch
      annotations:
        description: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentgenerationmismatch
        summary: Deployment generation mismatch due to possible roll-back
      expr: |-
        kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
          !=
        kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~".*"}
      for: 15m
      labels:
        severity: warning
    - alert: KubeDeploymentReplicasMismatch
      annotations:
        description: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentreplicasmismatch
        summary: Deployment has not matched the expected number of replicas.
      expr: |-
        (
          kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~".*"}
            >
          kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~".*"}
        ) and (
          changes(kube_deployment_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
            ==
          0
        )
      for: 15m
      labels:
        severity: warning
    - alert: KubeDeploymentRolloutStuck
      annotations:
        description: Rollout of deployment {{ $labels.namespace }}/{{ $labels.deployment }} is not progressing for longer than 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedeploymentrolloutstuck
        summary: Deployment rollout is not progressing.
      expr: |-
        kube_deployment_status_condition{condition="Progressing", status="false",job="kube-state-metrics", namespace=~".*"}
        != 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeStatefulSetReplicasMismatch
      annotations:
        description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetreplicasmismatch
        summary: StatefulSet has not matched the expected number of replicas.
      expr: |-
        (
          kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~".*"}
            !=
          kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~".*"}
        ) and (
          changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[10m])
            ==
          0
        )
      for: 15m
      labels:
        severity: warning
    - alert: KubeStatefulSetGenerationMismatch
      annotations:
        description: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetgenerationmismatch
        summary: StatefulSet generation mismatch due to possible roll-back
      expr: |-
        kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~".*"}
          !=
        kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~".*"}
      for: 15m
      labels:
        severity: warning
    - alert: KubeStatefulSetUpdateNotRolledOut
      annotations:
        description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubestatefulsetupdatenotrolledout
        summary: StatefulSet update has not been rolled out.
      expr: |-
        (
          max without (revision) (
            kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~".*"}
              unless
            kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~".*"}
          )
            *
          (
            kube_statefulset_replicas{job="kube-state-metrics", namespace=~".*"}
              !=
            kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}
          )
        )  and (
          changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~".*"}[5m])
            ==
          0
        )
      for: 15m
      labels:
        severity: warning
    - alert: KubeDaemonSetRolloutStuck
      annotations:
        description: DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetrolloutstuck
        summary: DaemonSet rollout is stuck.
      expr: |-
        (
          (
            kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"}
             !=
            kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
          ) or (
            kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"}
             !=
            0
          ) or (
            kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}
             !=
            kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
          ) or (
            kube_daemonset_status_number_available{job="kube-state-metrics", namespace=~".*"}
             !=
            kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
          )
        ) and (
          changes(kube_daemonset_status_updated_number_scheduled{job="kube-state-metrics", namespace=~".*"}[5m])
            ==
          0
        )
      for: 15m
      labels:
        severity: warning
    - alert: KubeContainerWaiting
      annotations:
        description: pod/{{ $labels.pod }} in namespace {{ $labels.namespace }} on container {{ $labels.container}} has been in waiting state for longer than 1 hour.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontainerwaiting
        summary: Pod container waiting longer than 1 hour
      expr: sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{job="kube-state-metrics", namespace=~".*"}) > 0
      for: 1h
      labels:
        severity: warning
    - alert: KubeDaemonSetNotScheduled
      annotations:
        description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetnotscheduled
        summary: DaemonSet pods are not scheduled.
      expr: |-
        kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~".*"}
          -
        kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~".*"} > 0
      for: 10m
      labels:
        severity: warning
    - alert: KubeDaemonSetMisScheduled
      annotations:
        description: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubedaemonsetmisscheduled
        summary: DaemonSet pods are misscheduled.
      expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~".*"} > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeJobNotCompleted
      annotations:
        description: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than {{ "43200" | humanizeDuration }} to complete.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobnotcompleted
        summary: Job did not complete in time
      expr: |-
        time() - max by (namespace, job_name, cluster) (kube_job_status_start_time{job="kube-state-metrics", namespace=~".*"}
          and
        kube_job_status_active{job="kube-state-metrics", namespace=~".*"} > 0) > 43200
      labels:
        severity: warning
    - alert: KubeJobFailed
      annotations:
        description: Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubejobfailed
        summary: Job failed to complete.
      expr: kube_job_failed{job="kube-state-metrics", namespace=~".*"}  > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeHpaReplicasMismatch
      annotations:
        description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has not matched the desired number of replicas for longer than 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpareplicasmismatch
        summary: HPA has not matched desired number of replicas.
      expr: |-
        (kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics", namespace=~".*"}
          !=
        kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"})
          and
        (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
          >
        kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics", namespace=~".*"})
          and
        (kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
          <
        kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"})
          and
        changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}[15m]) == 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeHpaMaxedOut
      annotations:
        description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler  }} has been running at max replicas for longer than 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubehpamaxedout
        summary: HPA is running at max replicas
      expr: |-
        kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics", namespace=~".*"}
          ==
        kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics", namespace=~".*"}
      for: 15m
      labels:
        severity: warning
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-resources.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubernetes-resources
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubernetes-resources
    rules:
    - alert: KubeCPUOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }} has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuovercommit
        summary: Cluster has overcommitted CPU resource requests.
      expr: |-
        sum(namespace_cpu:kube_pod_container_resource_requests:sum{job="kube-state-metrics",}) by (cluster) - (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
        and
        (sum(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster) - max(kube_node_status_allocatable{job="kube-state-metrics",resource="cpu"}) by (cluster)) > 0
      for: 10m
      labels:
        severity: warning
    - alert: KubeMemoryOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }} has overcommitted memory resource requests for Pods by {{ $value | humanize }} bytes and cannot tolerate node failure.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryovercommit
        summary: Cluster has overcommitted memory resource requests.
      expr: |-
        sum(namespace_memory:kube_pod_container_resource_requests:sum{}) by (cluster) - (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
        and
        (sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster) - max(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)) > 0
      for: 10m
      labels:
        severity: warning
    - alert: KubeCPUQuotaOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }}  has overcommitted CPU resource requests for Namespaces.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecpuquotaovercommit
        summary: Cluster has overcommitted CPU resource requests.
      expr: |-
        sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) by (cluster)
          /
        sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) by (cluster)
          > 1.5
      for: 5m
      labels:
        severity: warning
    - alert: KubeMemoryQuotaOvercommit
      annotations:
        description: Cluster {{ $labels.cluster }}  has overcommitted memory resource requests for Namespaces.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubememoryquotaovercommit
        summary: Cluster has overcommitted memory resource requests.
      expr: |-
        sum(min without(resource) (kube_resourcequota{job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) by (cluster)
          /
        sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) by (cluster)
          > 1.5
      for: 5m
      labels:
        severity: warning
    - alert: KubeQuotaAlmostFull
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaalmostfull
        summary: Namespace quota is going to be full.
      expr: |-
        kube_resourcequota{job="kube-state-metrics", type="used"}
          / ignoring(instance, job, type)
        (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
          > 0.9 < 1
      for: 15m
      labels:
        severity: info
    - alert: KubeQuotaFullyUsed
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotafullyused
        summary: Namespace quota is fully used.
      expr: |-
        kube_resourcequota{job="kube-state-metrics", type="used"}
          / ignoring(instance, job, type)
        (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
          == 1
      for: 15m
      labels:
        severity: info
    - alert: KubeQuotaExceeded
      annotations:
        description: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubequotaexceeded
        summary: Namespace quota has exceeded the limits.
      expr: |-
        kube_resourcequota{job="kube-state-metrics", type="used"}
          / ignoring(instance, job, type)
        (kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
          > 1
      for: 15m
      labels:
        severity: warning
    - alert: CPUThrottlingHigh
      annotations:
        description: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/cputhrottlinghigh
        summary: Processes experience elevated CPU throttling.
      expr: |-
        sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (cluster, container, pod, namespace)
          /
        sum(increase(container_cpu_cfs_periods_total{}[5m])) by (cluster, container, pod, namespace)
          > ( 25 / 100 )
      for: 15m
      labels:
        severity: info
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-storage.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubernetes-storage
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubernetes-storage
    rules:
    - alert: KubePersistentVolumeFillingUp
      annotations:
        description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is only {{ $value | humanizePercentage }} free.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
        summary: PersistentVolume is filling up.
      expr: |-
        (
          kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
            /
          kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
        ) < 0.03
        and
        kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
        unless on (cluster, namespace, persistentvolumeclaim)
        kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
        unless on (cluster, namespace, persistentvolumeclaim)
        kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
      for: 1m
      labels:
        severity: critical
    - alert: KubePersistentVolumeFillingUp
      annotations:
        description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
        summary: PersistentVolume is filling up.
      expr: |-
        (
          kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
            /
          kubelet_volume_stats_capacity_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
        ) < 0.15
        and
        kubelet_volume_stats_used_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
        and
        predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
        unless on (cluster, namespace, persistentvolumeclaim)
        kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
        unless on (cluster, namespace, persistentvolumeclaim)
        kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
      for: 1h
      labels:
        severity: warning
    - alert: KubePersistentVolumeInodesFillingUp
      annotations:
        description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} only has {{ $value | humanizePercentage }} free inodes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
        summary: PersistentVolumeInodes are filling up.
      expr: |-
        (
          kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}
            /
          kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
        ) < 0.03
        and
        kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
        unless on (cluster, namespace, persistentvolumeclaim)
        kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
        unless on (cluster, namespace, persistentvolumeclaim)
        kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
      for: 1m
      labels:
        severity: critical
    - alert: KubePersistentVolumeInodesFillingUp
      annotations:
        description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeinodesfillingup
        summary: PersistentVolumeInodes are filling up.
      expr: |-
        (
          kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}
            /
          kubelet_volume_stats_inodes{job="kubelet", namespace=~".*", metrics_path="/metrics"}
        ) < 0.15
        and
        kubelet_volume_stats_inodes_used{job="kubelet", namespace=~".*", metrics_path="/metrics"} > 0
        and
        predict_linear(kubelet_volume_stats_inodes_free{job="kubelet", namespace=~".*", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
        unless on (cluster, namespace, persistentvolumeclaim)
        kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
        unless on (cluster, namespace, persistentvolumeclaim)
        kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
      for: 1h
      labels:
        severity: warning
    - alert: KubePersistentVolumeErrors
      annotations:
        description: The persistent volume {{ $labels.persistentvolume }} {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} has status {{ $labels.phase }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumeerrors
        summary: PersistentVolume is having issues with provisioning.
      expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
      for: 5m
      labels:
        severity: critical
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system-apiserver.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubernetes-system-apiserver
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubernetes-system-apiserver
    rules:
    - alert: KubeClientCertificateExpiration
      annotations:
        description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 7.0 days.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
        summary: Client certificate is about to expire.
      expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800
      for: 5m
      labels:
        severity: warning
    - alert: KubeClientCertificateExpiration
      annotations:
        description: A client certificate used to authenticate to kubernetes apiserver is expiring in less than 24.0 hours.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclientcertificateexpiration
        summary: Client certificate is about to expire.
      expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on (job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
      for: 5m
      labels:
        severity: critical
    - alert: KubeAggregatedAPIErrors
      annotations:
        description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapierrors
        summary: Kubernetes aggregated API has reported errors.
      expr: sum by (name, namespace, cluster)(increase(aggregator_unavailable_apiservice_total{job="apiserver"}[10m])) > 4
      labels:
        severity: warning
    - alert: KubeAggregatedAPIDown
      annotations:
        description: Kubernetes aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeaggregatedapidown
        summary: Kubernetes aggregated API is down.
      expr: (1 - max by (name, namespace, cluster)(avg_over_time(aggregator_unavailable_apiservice{job="apiserver"}[10m]))) * 100 < 85
      for: 5m
      labels:
        severity: warning
    - alert: KubeAPIDown
      annotations:
        description: KubeAPI has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapidown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="apiserver"} == 1)
      for: 15m
      labels:
        severity: critical
    - alert: KubeAPITerminatedRequests
      annotations:
        description: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeapiterminatedrequests
        summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
      expr: sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m]))  / (  sum(rate(apiserver_request_total{job="apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="apiserver"}[10m])) ) > 0.20
      for: 5m
      labels:
        severity: warning
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system-controller-manager.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubernetes-system-controller-manager
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubernetes-system-controller-manager
    rules:
    - alert: KubeControllerManagerDown
      annotations:
        description: KubeControllerManager has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubecontrollermanagerdown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="kube-controller-manager"} == 1)
      for: 15m
      labels:
        severity: critical
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system-kubelet.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubernetes-system-kubelet
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubernetes-system-kubelet
    rules:
    - alert: KubeNodeNotReady
      annotations:
        description: '{{ $labels.node }} has been unready for more than 15 minutes.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodenotready
        summary: Node is not ready.
      expr: kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeNodeUnreachable
      annotations:
        description: '{{ $labels.node }} is unreachable and some workloads may be rescheduled.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodeunreachable
        summary: Node is unreachable.
      expr: (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
      for: 15m
      labels:
        severity: warning
    - alert: KubeletTooManyPods
      annotations:
        description: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubelettoomanypods
        summary: Kubelet is running at capacity.
      expr: |-
        count by (cluster, node) (
          (kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on (instance,pod,namespace,cluster) group_left(node) topk by (instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
        )
        /
        max by (cluster, node) (
          kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
        ) > 0.95
      for: 15m
      labels:
        severity: info
    - alert: KubeNodeReadinessFlapping
      annotations:
        description: The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubenodereadinessflapping
        summary: Node readiness status is flapping.
      expr: sum(changes(kube_node_status_condition{job="kube-state-metrics",status="true",condition="Ready"}[15m])) by (cluster, node) > 2
      for: 15m
      labels:
        severity: warning
    - alert: KubeletPlegDurationHigh
      annotations:
        description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletplegdurationhigh
        summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
      expr: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
      for: 5m
      labels:
        severity: warning
    - alert: KubeletPodStartUpLatencyHigh
      annotations:
        description: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletpodstartuplatencyhigh
        summary: Kubelet Pod startup latency is too high.
      expr: histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (cluster, instance, le)) * on (cluster, instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
      for: 15m
      labels:
        severity: warning
    - alert: KubeletClientCertificateExpiration
      annotations:
        description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration
        summary: Kubelet client certificate is about to expire.
      expr: kubelet_certificate_manager_client_ttl_seconds < 604800
      labels:
        severity: warning
    - alert: KubeletClientCertificateExpiration
      annotations:
        description: Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificateexpiration
        summary: Kubelet client certificate is about to expire.
      expr: kubelet_certificate_manager_client_ttl_seconds < 86400
      labels:
        severity: critical
    - alert: KubeletServerCertificateExpiration
      annotations:
        description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration
        summary: Kubelet server certificate is about to expire.
      expr: kubelet_certificate_manager_server_ttl_seconds < 604800
      labels:
        severity: warning
    - alert: KubeletServerCertificateExpiration
      annotations:
        description: Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificateexpiration
        summary: Kubelet server certificate is about to expire.
      expr: kubelet_certificate_manager_server_ttl_seconds < 86400
      labels:
        severity: critical
    - alert: KubeletClientCertificateRenewalErrors
      annotations:
        description: Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes).
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletclientcertificaterenewalerrors
        summary: Kubelet has failed to renew its client certificate.
      expr: increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeletServerCertificateRenewalErrors
      annotations:
        description: Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletservercertificaterenewalerrors
        summary: Kubelet has failed to renew its server certificate.
      expr: increase(kubelet_server_expiration_renew_errors[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: KubeletDown
      annotations:
        description: Kubelet has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeletdown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="kubelet", metrics_path="/metrics"} == 1)
      for: 15m
      labels:
        severity: critical
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system-scheduler.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubernetes-system-scheduler
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubernetes-system-scheduler
    rules:
    - alert: KubeSchedulerDown
      annotations:
        description: KubeScheduler has disappeared from Prometheus target discovery.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeschedulerdown
        summary: Target disappeared from Prometheus target discovery.
      expr: absent(up{job="kube-scheduler"} == 1)
      for: 15m
      labels:
        severity: critical
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/kubernetes-system.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-kubernetes-system
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: kubernetes-system
    rules:
    - alert: KubeVersionMismatch
      annotations:
        description: There are {{ $value }} different semantic versions of Kubernetes components running.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeversionmismatch
        summary: Different semantic versions of Kubernetes components running.
      expr: count by (cluster) (count by (git_version, cluster) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1
      for: 15m
      labels:
        severity: warning
    - alert: KubeClientErrors
      annotations:
        description: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubeclienterrors
        summary: Kubernetes API server client is experiencing errors.
      expr: |-
        (sum(rate(rest_client_requests_total{job="apiserver",code=~"5.."}[5m])) by (cluster, instance, job, namespace)
          /
        sum(rate(rest_client_requests_total{job="apiserver"}[5m])) by (cluster, instance, job, namespace))
        > 0.01
      for: 15m
      labels:
        severity: warning
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/node-exporter.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-node-exporter.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: node-exporter.rules
    rules:
    - expr: |-
        count without (cpu, mode) (
          node_cpu_seconds_total{job="node-exporter",mode="idle"}
        )
      record: instance:node_num_cpu:sum
    - expr: |-
        1 - avg without (cpu) (
          sum without (mode) (rate(node_cpu_seconds_total{job="node-exporter", mode=~"idle|iowait|steal"}[5m]))
        )
      record: instance:node_cpu_utilisation:rate5m
    - expr: |-
        (
          node_load1{job="node-exporter"}
        /
          instance:node_num_cpu:sum{job="node-exporter"}
        )
      record: instance:node_load1_per_cpu:ratio
    - expr: |-
        1 - (
          (
            node_memory_MemAvailable_bytes{job="node-exporter"}
            or
            (
              node_memory_Buffers_bytes{job="node-exporter"}
              +
              node_memory_Cached_bytes{job="node-exporter"}
              +
              node_memory_MemFree_bytes{job="node-exporter"}
              +
              node_memory_Slab_bytes{job="node-exporter"}
            )
          )
        /
          node_memory_MemTotal_bytes{job="node-exporter"}
        )
      record: instance:node_memory_utilisation:ratio
    - expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m])
      record: instance:node_vmstat_pgmajfault:rate5m
    - expr: rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
      record: instance_device:node_disk_io_time_seconds:rate5m
    - expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m])
      record: instance_device:node_disk_io_time_weighted_seconds:rate5m
    - expr: |-
        sum without (device) (
          rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[5m])
        )
      record: instance:node_network_receive_bytes_excluding_lo:rate5m
    - expr: |-
        sum without (device) (
          rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[5m])
        )
      record: instance:node_network_transmit_bytes_excluding_lo:rate5m
    - expr: |-
        sum without (device) (
          rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[5m])
        )
      record: instance:node_network_receive_drop_excluding_lo:rate5m
    - expr: |-
        sum without (device) (
          rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[5m])
        )
      record: instance:node_network_transmit_drop_excluding_lo:rate5m
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/node-exporter.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-node-exporter
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: node-exporter
    rules:
    - alert: NodeFilesystemSpaceFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
        summary: Filesystem is predicted to run out of space within the next 24 hours.
      expr: |-
        (
          node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 15
        and
          predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
        and
          node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemSpaceFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
        summary: Filesystem is predicted to run out of space within the next 4 hours.
      expr: |-
        (
          node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 10
        and
          predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
        and
          node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: critical
    - alert: NodeFilesystemAlmostOutOfSpace
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
        summary: Filesystem has less than 5% space left.
      expr: |-
        (
          node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
        and
          node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 30m
      labels:
        severity: warning
    - alert: NodeFilesystemAlmostOutOfSpace
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
        summary: Filesystem has less than 3% space left.
      expr: |-
        (
          node_filesystem_avail_bytes{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
        and
          node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 30m
      labels:
        severity: critical
    - alert: NodeFilesystemFilesFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
        summary: Filesystem is predicted to run out of inodes within the next 24 hours.
      expr: |-
        (
          node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 40
        and
          predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 24*60*60) < 0
        and
          node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemFilesFillingUp
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
        summary: Filesystem is predicted to run out of inodes within the next 4 hours.
      expr: |-
        (
          node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 20
        and
          predict_linear(node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""}[6h], 4*60*60) < 0
        and
          node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: critical
    - alert: NodeFilesystemAlmostOutOfFiles
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
        summary: Filesystem has less than 5% inodes left.
      expr: |-
        (
          node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 5
        and
          node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: warning
    - alert: NodeFilesystemAlmostOutOfFiles
      annotations:
        description: Filesystem on {{ $labels.device }}, mounted on {{ $labels.mountpoint }}, at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
        summary: Filesystem has less than 3% inodes left.
      expr: |-
        (
          node_filesystem_files_free{job="node-exporter",fstype!="",mountpoint!=""} / node_filesystem_files{job="node-exporter",fstype!="",mountpoint!=""} * 100 < 3
        and
          node_filesystem_readonly{job="node-exporter",fstype!="",mountpoint!=""} == 0
        )
      for: 1h
      labels:
        severity: critical
    - alert: NodeNetworkReceiveErrs
      annotations:
        description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs
        summary: Network interface is reporting many receive errors.
      expr: rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01
      for: 1h
      labels:
        severity: warning
    - alert: NodeNetworkTransmitErrs
      annotations:
        description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs
        summary: Network interface is reporting many transmit errors.
      expr: rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01
      for: 1h
      labels:
        severity: warning
    - alert: NodeHighNumberConntrackEntriesUsed
      annotations:
        description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused
        summary: Number of conntrack are getting close to the limit.
      expr: (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75
      labels:
        severity: warning
    - alert: NodeTextFileCollectorScrapeError
      annotations:
        description: Node Exporter text file collector on {{ $labels.instance }} failed to scrape.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror
        summary: Node Exporter text file collector failed to scrape.
      expr: node_textfile_scrape_error{job="node-exporter"} == 1
      labels:
        severity: warning
    - alert: NodeClockSkewDetected
      annotations:
        description: Clock at {{ $labels.instance }} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected
        summary: Clock skew detected.
      expr: |-
        (
          node_timex_offset_seconds{job="node-exporter"} > 0.05
        and
          deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) >= 0
        )
        or
        (
          node_timex_offset_seconds{job="node-exporter"} < -0.05
        and
          deriv(node_timex_offset_seconds{job="node-exporter"}[5m]) <= 0
        )
      for: 10m
      labels:
        severity: warning
    - alert: NodeClockNotSynchronising
      annotations:
        description: Clock at {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising
        summary: Clock not synchronising.
      expr: |-
        min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0
        and
        node_timex_maxerror_seconds{job="node-exporter"} >= 16
      for: 10m
      labels:
        severity: warning
    - alert: NodeRAIDDegraded
      annotations:
        description: RAID array '{{ $labels.device }}' at {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded
        summary: RAID Array is degraded.
      expr: node_md_disks_required{job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}) > 0
      for: 15m
      labels:
        severity: critical
    - alert: NodeRAIDDiskFailure
      annotations:
        description: At least one device in RAID array at {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure
        summary: Failed device in RAID array.
      expr: node_md_disks{state="failed",job="node-exporter",device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"} > 0
      labels:
        severity: warning
    - alert: NodeFileDescriptorLimit
      annotations:
        description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
        summary: Kernel is predicted to exhaust file descriptors limit soon.
      expr: |-
        (
          node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 70
        )
      for: 15m
      labels:
        severity: warning
    - alert: NodeFileDescriptorLimit
      annotations:
        description: File descriptors limit at {{ $labels.instance }} is currently at {{ printf "%.2f" $value }}%.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
        summary: Kernel is predicted to exhaust file descriptors limit soon.
      expr: |-
        (
          node_filefd_allocated{job="node-exporter"} * 100 / node_filefd_maximum{job="node-exporter"} > 90
        )
      for: 15m
      labels:
        severity: critical
    - alert: NodeCPUHighUsage
      annotations:
        description: 'CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.

          '
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodecpuhighusage
        summary: High CPU usage.
      expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90
      for: 15m
      labels:
        severity: info
    - alert: NodeSystemSaturation
      annotations:
        description: 'System load per core at {{ $labels.instance }} has been above 2 for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.

          This might indicate this instance resources saturation and can cause it becoming unresponsive.

          '
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemsaturation
        summary: System saturated, load per core is very high.
      expr: |-
        node_load1{job="node-exporter"}
        / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2
      for: 15m
      labels:
        severity: warning
    - alert: NodeMemoryMajorPagesFaults
      annotations:
        description: 'Memory major pages are occurring at very high rate at {{ $labels.instance }}, 500 major page faults per second for the last 15 minutes, is currently at {{ printf "%.2f" $value }}.

          Please check that there is enough memory available at this instance.

          '
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememorymajorpagesfaults
        summary: Memory major page faults are occurring at very high rate.
      expr: rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500
      for: 15m
      labels:
        severity: warning
    - alert: NodeMemoryHighUtilization
      annotations:
        description: 'Memory is filling up at {{ $labels.instance }}, has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.

          '
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodememoryhighutilization
        summary: Host is running out of memory.
      expr: 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90
      for: 15m
      labels:
        severity: warning
    - alert: NodeDiskIOSaturation
      annotations:
        description: 'Disk IO queue (aqu-sq) is high on {{ $labels.device }} at {{ $labels.instance }}, has been above 10 for the last 30 minutes, is currently at {{ printf "%.2f" $value }}.

          This symptom might indicate disk saturation.

          '
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodediskiosaturation
        summary: Disk IO queue is high.
      expr: rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"(/dev/)?(mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|md.+|dasd.+)"}[5m]) > 10
      for: 30m
      labels:
        severity: warning
    - alert: NodeSystemdServiceFailed
      annotations:
        description: Systemd service {{ $labels.name }} has entered failed state at {{ $labels.instance }}
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodesystemdservicefailed
        summary: Systemd service has entered failed state.
      expr: node_systemd_unit_state{job="node-exporter", state="failed"} == 1
      for: 5m
      labels:
        severity: warning
    - alert: NodeBondingDegraded
      annotations:
        description: Bonding interface {{ $labels.master }} on {{ $labels.instance }} is in degraded state due to one or more slave failures.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodebondingdegraded
        summary: Bonding interface is degraded
      expr: (node_bonding_slaves - node_bonding_active) != 0
      for: 5m
      labels:
        severity: warning
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/node-network.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-node-network
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: node-network
    rules:
    - alert: NodeNetworkInterfaceFlapping
      annotations:
        description: Network interface "{{ $labels.device }}" changing its up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/nodenetworkinterfaceflapping
        summary: Network interface is often changing its status
      expr: changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2
      for: 2m
      labels:
        severity: warning
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/node.rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-node.rules
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: node.rules
    rules:
    - expr: |-
        topk by (cluster, namespace, pod) (1,
          max by (cluster, node, namespace, pod) (
            label_replace(kube_pod_info{job="kube-state-metrics",node!=""}, "pod", "$1", "pod", "(.*)")
        ))
      record: 'node_namespace_pod:kube_pod_info:'
    - expr: |-
        count by (cluster, node) (
          node_cpu_seconds_total{mode="idle",job="node-exporter"}
          * on (cluster, namespace, pod) group_left(node)
          topk by (cluster, namespace, pod) (1, node_namespace_pod:kube_pod_info:)
        )
      record: node:node_num_cpu:sum
    - expr: |-
        sum(
          node_memory_MemAvailable_bytes{job="node-exporter"} or
          (
            node_memory_Buffers_bytes{job="node-exporter"} +
            node_memory_Cached_bytes{job="node-exporter"} +
            node_memory_MemFree_bytes{job="node-exporter"} +
            node_memory_Slab_bytes{job="node-exporter"}
          )
        ) by (cluster)
      record: :node_memory_MemAvailable_bytes:sum
    - expr: |-
        avg by (cluster, node) (
          sum without (mode) (
            rate(node_cpu_seconds_total{mode!="idle",mode!="iowait",mode!="steal",job="node-exporter"}[5m])
          )
        )
      record: node:node_cpu_utilization:ratio_rate5m
    - expr: |-
        avg by (cluster) (
          node:node_cpu_utilization:ratio_rate5m
        )
      record: cluster:node_cpu:ratio_rate5m
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/prometheus-operator.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-prometheus-operator
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: prometheus-operator
    rules:
    - alert: PrometheusOperatorListErrors
      annotations:
        description: Errors while performing List operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorlisterrors
        summary: Errors while performing list operations in controller.
      expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_failed_total{job="release-name-kps-operator",namespace="default"}[10m])) / sum by (cluster,controller,namespace) (rate(prometheus_operator_list_operations_total{job="release-name-kps-operator",namespace="default"}[10m]))) > 0.4
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusOperatorWatchErrors
      annotations:
        description: Errors while performing watch operations in controller {{$labels.controller}} in {{$labels.namespace}} namespace.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorwatcherrors
        summary: Errors while performing watch operations in controller.
      expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_failed_total{job="release-name-kps-operator",namespace="default"}[5m])) / sum by (cluster,controller,namespace) (rate(prometheus_operator_watch_operations_total{job="release-name-kps-operator",namespace="default"}[5m]))) > 0.4
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusOperatorSyncFailed
      annotations:
        description: Controller {{ $labels.controller }} in {{ $labels.namespace }} namespace fails to reconcile {{ $value }} objects.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorsyncfailed
        summary: Last controller reconciliation failed
      expr: min_over_time(prometheus_operator_syncs{status="failed",job="release-name-kps-operator",namespace="default"}[5m]) > 0
      for: 10m
      labels:
        severity: warning
    - alert: PrometheusOperatorReconcileErrors
      annotations:
        description: '{{ $value | humanizePercentage }} of reconciling operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorreconcileerrors
        summary: Errors while reconciling objects.
      expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_errors_total{job="release-name-kps-operator",namespace="default"}[5m]))) / (sum by (cluster,controller,namespace) (rate(prometheus_operator_reconcile_operations_total{job="release-name-kps-operator",namespace="default"}[5m]))) > 0.1
      for: 10m
      labels:
        severity: warning
    - alert: PrometheusOperatorStatusUpdateErrors
      annotations:
        description: '{{ $value | humanizePercentage }} of status update operations failed for {{ $labels.controller }} controller in {{ $labels.namespace }} namespace.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorstatusupdateerrors
        summary: Errors while updating objects status.
      expr: (sum by (cluster,controller,namespace) (rate(prometheus_operator_status_update_errors_total{job="release-name-kps-operator",namespace="default"}[5m]))) / (sum by (cluster,controller,namespace) (rate(prometheus_operator_status_update_operations_total{job="release-name-kps-operator",namespace="default"}[5m]))) > 0.1
      for: 10m
      labels:
        severity: warning
    - alert: PrometheusOperatorNodeLookupErrors
      annotations:
        description: Errors while reconciling Prometheus in {{ $labels.namespace }} Namespace.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornodelookuperrors
        summary: Errors while reconciling Prometheus.
      expr: rate(prometheus_operator_node_address_lookup_errors_total{job="release-name-kps-operator",namespace="default"}[5m]) > 0.1
      for: 10m
      labels:
        severity: warning
    - alert: PrometheusOperatorNotReady
      annotations:
        description: Prometheus operator in {{ $labels.namespace }} namespace isn't ready to reconcile {{ $labels.controller }} resources.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatornotready
        summary: Prometheus operator not ready
      expr: min by (cluster,controller,namespace) (max_over_time(prometheus_operator_ready{job="release-name-kps-operator",namespace="default"}[5m]) == 0)
      for: 5m
      labels:
        severity: warning
    - alert: PrometheusOperatorRejectedResources
      annotations:
        description: Prometheus operator in {{ $labels.namespace }} namespace rejected {{ printf "%0.0f" $value }} {{ $labels.controller }}/{{ $labels.resource }} resources.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus-operator/prometheusoperatorrejectedresources
        summary: Resources rejected by Prometheus operator
      expr: min_over_time(prometheus_operator_managed_resources{state="rejected",job="release-name-kps-operator",namespace="default"}[5m]) > 0
      for: 5m
      labels:
        severity: warning
---
# Source: mimir/charts/kps/templates/prometheus/rules-1.14/prometheus.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: release-name-kps-prometheus
  namespace: default
  labels:
    app: kps
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  groups:
  - name: prometheus
    rules:
    - alert: PrometheusBadConfig
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusbadconfig
        summary: Failed Prometheus configuration reload.
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        max_over_time(prometheus_config_last_reload_successful{job="release-name-kps-prometheus",namespace="default"}[5m]) == 0
      for: 10m
      labels:
        severity: critical
    - alert: PrometheusSDRefreshFailure
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to refresh SD with mechanism {{$labels.mechanism}}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheussdrefreshfailure
        summary: Failed Prometheus SD refresh.
      expr: increase(prometheus_sd_refresh_failures_total{job="release-name-kps-prometheus",namespace="default"}[10m]) > 0
      for: 20m
      labels:
        severity: warning
    - alert: PrometheusNotificationQueueRunningFull
      annotations:
        description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotificationqueuerunningfull
        summary: Prometheus alert notification queue predicted to run full in less than 30m.
      expr: |-
        # Without min_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        (
          predict_linear(prometheus_notifications_queue_length{job="release-name-kps-prometheus",namespace="default"}[5m], 60 * 30)
        >
          min_over_time(prometheus_notifications_queue_capacity{job="release-name-kps-prometheus",namespace="default"}[5m])
        )
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
      annotations:
        description: '{{ printf "%.1f" $value }}% errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuserrorsendingalertstosomealertmanagers
        summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.
      expr: |-
        (
          rate(prometheus_notifications_errors_total{job="release-name-kps-prometheus",namespace="default"}[5m])
        /
          rate(prometheus_notifications_sent_total{job="release-name-kps-prometheus",namespace="default"}[5m])
        )
        * 100
        > 1
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusNotConnectedToAlertmanagers
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotconnectedtoalertmanagers
        summary: Prometheus is not connected to any Alertmanagers.
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        max_over_time(prometheus_notifications_alertmanagers_discovered{job="release-name-kps-prometheus",namespace="default"}[5m]) < 1
      for: 10m
      labels:
        severity: warning
    - alert: PrometheusTSDBReloadsFailing
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbreloadsfailing
        summary: Prometheus has issues reloading blocks from disk.
      expr: increase(prometheus_tsdb_reloads_failures_total{job="release-name-kps-prometheus",namespace="default"}[3h]) > 0
      for: 4h
      labels:
        severity: warning
    - alert: PrometheusTSDBCompactionsFailing
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbcompactionsfailing
        summary: Prometheus has issues compacting blocks.
      expr: increase(prometheus_tsdb_compactions_failed_total{job="release-name-kps-prometheus",namespace="default"}[3h]) > 0
      for: 4h
      labels:
        severity: warning
    - alert: PrometheusNotIngestingSamples
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotingestingsamples
        summary: Prometheus is not ingesting samples.
      expr: |-
        (
          sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="release-name-kps-prometheus",namespace="default"}[5m])) <= 0
        and
          (
            sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="release-name-kps-prometheus",namespace="default"}) > 0
          or
            sum without(rule_group) (prometheus_rule_group_rules{job="release-name-kps-prometheus",namespace="default"}) > 0
          )
        )
      for: 10m
      labels:
        severity: warning
    - alert: PrometheusDuplicateTimestamps
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with different values but duplicated timestamp.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusduplicatetimestamps
        summary: Prometheus is dropping samples with duplicate timestamps.
      expr: rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
      for: 10m
      labels:
        severity: warning
    - alert: PrometheusOutOfOrderTimestamps
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with timestamps arriving out of order.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusoutofordertimestamps
        summary: Prometheus drops samples with out-of-order timestamps.
      expr: rate(prometheus_target_scrapes_sample_out_of_order_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
      for: 10m
      labels:
        severity: warning
    - alert: PrometheusRemoteStorageFailures
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotestoragefailures
        summary: Prometheus fails to send samples to remote storage.
      expr: |-
        (
          (rate(prometheus_remote_storage_failed_samples_total{job="release-name-kps-prometheus",namespace="default"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="release-name-kps-prometheus",namespace="default"}[5m]))
        /
          (
            (rate(prometheus_remote_storage_failed_samples_total{job="release-name-kps-prometheus",namespace="default"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="release-name-kps-prometheus",namespace="default"}[5m]))
          +
            (rate(prometheus_remote_storage_succeeded_samples_total{job="release-name-kps-prometheus",namespace="default"}[5m]) or rate(prometheus_remote_storage_samples_total{job="release-name-kps-prometheus",namespace="default"}[5m]))
          )
        )
        * 100
        > 1
      for: 15m
      labels:
        severity: critical
    - alert: PrometheusRemoteWriteBehind
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritebehind
        summary: Prometheus remote write is behind.
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        (
          max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="release-name-kps-prometheus",namespace="default"}[5m])
        - ignoring(remote_name, url) group_right
          max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="release-name-kps-prometheus",namespace="default"}[5m])
        )
        > 120
      for: 15m
      labels:
        severity: critical
    - alert: PrometheusRemoteWriteDesiredShards
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="release-name-kps-prometheus",namespace="default"}` $labels.instance | query | first | value }}.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritedesiredshards
        summary: Prometheus remote write desired shards calculation wants to run more than configured max shards.
      expr: |-
        # Without max_over_time, failed scrapes could create false negatives, see
        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
        (
          max_over_time(prometheus_remote_storage_shards_desired{job="release-name-kps-prometheus",namespace="default"}[5m])
        >
          max_over_time(prometheus_remote_storage_shards_max{job="release-name-kps-prometheus",namespace="default"}[5m])
        )
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusRuleFailures
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusrulefailures
        summary: Prometheus is failing rule evaluations.
      expr: increase(prometheus_rule_evaluation_failures_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
      for: 15m
      labels:
        severity: critical
    - alert: PrometheusMissingRuleEvaluations
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusmissingruleevaluations
        summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
      expr: increase(prometheus_rule_group_iterations_missed_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusTargetLimitHit
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because the number of targets exceeded the configured target_limit.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetlimithit
        summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit.
      expr: increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusLabelLimitHit
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuslabellimithit
        summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit.
      expr: increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusScrapeBodySizeLimitHit
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapebodysizelimithit
        summary: Prometheus has dropped some targets that exceeded body size limit.
      expr: increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusScrapeSampleLimitHit
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapesamplelimithit
        summary: Prometheus has failed scrapes that have exceeded the configured sample limit.
      expr: increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusTargetSyncFailure
      annotations:
        description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.namespace}}/{{$labels.pod}} have failed to sync because invalid configuration was supplied.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetsyncfailure
        summary: Prometheus has failed to sync targets.
      expr: increase(prometheus_target_sync_failed_total{job="release-name-kps-prometheus",namespace="default"}[30m]) > 0
      for: 5m
      labels:
        severity: critical
    - alert: PrometheusHighQueryLoad
      annotations:
        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} query API has less than 20% available capacity in its query engine for the last 15 minutes.
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheushighqueryload
        summary: Prometheus is reaching its maximum capacity serving concurrent requests.
      expr: avg_over_time(prometheus_engine_queries{job="release-name-kps-prometheus",namespace="default"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="release-name-kps-prometheus",namespace="default"}[5m]) > 0.8
      for: 15m
      labels:
        severity: warning
    - alert: PrometheusErrorSendingAlertsToAnyAlertmanager
      annotations:
        description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuserrorsendingalertstoanyalertmanager
        summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
      expr: |-
        min without (alertmanager) (
          rate(prometheus_notifications_errors_total{job="release-name-kps-prometheus",namespace="default",alertmanager!~``}[5m])
        /
          rate(prometheus_notifications_sent_total{job="release-name-kps-prometheus",namespace="default",alertmanager!~``}[5m])
        )
        * 100
        > 3
      for: 15m
      labels:
        severity: critical
---
# Source: mimir/charts/kps/charts/kube-state-metrics/templates/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-kube-state-metrics
  namespace: default
  labels:    
    helm.sh/chart: kube-state-metrics-5.16.0
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: kube-state-metrics
    app.kubernetes.io/name: kube-state-metrics
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "2.10.1"
    release: release-name
spec:
  jobLabel: app.kubernetes.io/name  
  selector:
    matchLabels:      
      app.kubernetes.io/name: kube-state-metrics
      app.kubernetes.io/instance: release-name
  endpoints:
    - port: http
      honorLabels: true
---
# Source: mimir/charts/kps/charts/prometheus-node-exporter/templates/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-prometheus-node-exporter
  namespace: default
  labels:
    helm.sh/chart: prometheus-node-exporter-4.30.2
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/component: metrics
    app.kubernetes.io/part-of: prometheus-node-exporter
    app.kubernetes.io/name: prometheus-node-exporter
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "1.7.0"
    jobLabel: node-exporter
    release: release-name
spec:
  jobLabel: jobLabel
  
  selector:
    matchLabels:
      app.kubernetes.io/name: prometheus-node-exporter
      app.kubernetes.io/instance: release-name
  attachMetadata:
    node: false
  endpoints:
    - port: http-metrics
      scheme: http
---
# Source: mimir/charts/kps/templates/exporters/core-dns/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-kps-coredns
  namespace: default
  labels:
    app: kps-coredns
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  jobLabel: jobLabel
  
  selector:
    matchLabels:
      app: kps-coredns
      release: "release-name"
  namespaceSelector:
    matchNames:
      - "kube-system"
  endpoints:
  - port: http-metrics
    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
---
# Source: mimir/charts/kps/templates/exporters/kube-api-server/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-kps-apiserver
  namespace: default
  labels:
    app: kps-apiserver
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    port: https
    scheme: https
    metricRelabelings:
      - action: drop
        regex: apiserver_request_duration_seconds_bucket;(0.15|0.2|0.3|0.35|0.4|0.45|0.6|0.7|0.8|0.9|1.25|1.5|1.75|2|3|3.5|4|4.5|6|7|8|9|15|25|40|50)
        sourceLabels:
        - __name__
        - le
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      serverName: kubernetes
      insecureSkipVerify: false
  jobLabel: component
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      component: apiserver
      provider: kubernetes
---
# Source: mimir/charts/kps/templates/exporters/kube-controller-manager/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-kps-kube-controller-manager
  namespace: default
  labels:
    app: kps-kube-controller-manager
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  jobLabel: jobLabel
  
  selector:
    matchLabels:
      app: kps-kube-controller-manager
      release: "release-name"
  namespaceSelector:
    matchNames:
      - "kube-system"
  endpoints:
  - port: http-metrics
    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    scheme: https
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecureSkipVerify: true
---
# Source: mimir/charts/kps/templates/exporters/kube-etcd/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-kps-kube-etcd
  namespace: default
  labels:
    app: kps-kube-etcd
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  jobLabel: jobLabel
    
  selector:
    matchLabels:
      app: kps-kube-etcd
      release: "release-name"
  namespaceSelector:
    matchNames:
      - "kube-system"
  endpoints:
  - port: http-metrics
    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
---
# Source: mimir/charts/kps/templates/exporters/kube-scheduler/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-kps-kube-scheduler
  namespace: default
  labels:
    app: kps-kube-scheduler
    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  jobLabel: jobLabel
  
  selector:
    matchLabels:
      app: kps-kube-scheduler
      release: "release-name"
  namespaceSelector:
    matchNames:
      - "kube-system"
  endpoints:
  - port: http-metrics
    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    scheme: https
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecureSkipVerify: true
---
# Source: mimir/charts/kps/templates/exporters/kubelet/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-kps-kubelet
  namespace: default
  labels:
    app: kps-kubelet    
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/version: "56.9.0"
    app.kubernetes.io/part-of: kps
    chart: kps-56.9.0
    release: "release-name"
    heritage: "Helm"
spec:
  
  attachMetadata:
    node: false
  endpoints:
  - port: https-metrics
    scheme: https
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecureSkipVerify: true
    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    honorLabels: true
    honorTimestamps: true
    relabelings:
    - action: replace
      sourceLabels:
      - __metrics_path__
      targetLabel: metrics_path
  - port: https-metrics
    scheme: https
    path: /metrics/cadvisor
    honorLabels: true
    honorTimestamps: true
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecureSkipVerify: true
    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    metricRelabelings:
    - action: drop
      regex: container_cpu_(cfs_throttled_seconds_total|load_average_10s|system_seconds_total|user_seconds_total)
      sourceLabels:
      - __name__
    - action: drop
      regex: container_fs_(io_current|io_time_seconds_total|io_time_weighted_seconds_total|reads_merged_total|sector_reads_total|sector_writes_total|writes_merged_total)
      sourceLabels:
      - __name__
    - action: drop
      regex: container_memory_(mapped_file|swap)
      sourceLabels:
      - __name__
    - action: drop
      regex: container_(file_descriptors|tasks_state|threads_max)
      sourceLabels:
      - __name__
    - action: drop
      regex: container_spec.*
      sourceLabels:
      - __name__
    - action: drop
      regex: .+;
      sourceLabels:
      - id
      - pod
    relabelings:
    - action: replace
      sourceLabels:
      - __metrics_path__
      targetLabel: metrics_path
  - port: https-metrics
    scheme: https
    path: /metrics/probes
    honorLabels: true
    honorTimestamps: true
    tlsConfig:
      caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      insecureSkipVerify: true
    bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabelings:
    - action: replace
      sourceLabels:
      - __metrics_path__
      targetLabel: metrics_path
  jobLabel: k8s-app
  namespaceSelector:
    matchNames:
    - kube-system
  selector:
    matchLabels:
      app.kubernetes.io/name: kubelet
      k8s-app: kubelet
---
# Source: mimir/charts/mimir/templates/alertmanager/alertmanager-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-alertmanager
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: alertmanager
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: alertmanager
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/alertmanager"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/compactor/compactor-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-compactor
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: compactor
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: compactor
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/compactor"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/distributor/distributor-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-distributor
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: distributor
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: distributor
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/distributor"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/index-cache/index-cache-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-index-cache
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: index-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: index-cache
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/index-cache"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/ingester/ingester-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-ingester
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ingester
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: ingester
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/ingester"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/metadata-cache/metadata-cache-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-metadata-cache
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: metadata-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: metadata-cache
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/metadata-cache"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/overrides-exporter/overrides-exporter-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-overrides-exporter
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: overrides-exporter
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: overrides-exporter
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/overrides-exporter"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/querier/querier-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-querier
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: querier
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: querier
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/querier"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/query-frontend/query-frontend-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-query-frontend
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-frontend
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: query-frontend
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/query-frontend"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/query-scheduler/query-scheduler-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-query-scheduler
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: query-scheduler
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: query-scheduler
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/query-scheduler"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/results-cache/results-cache-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-results-cache
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: results-cache
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: results-cache
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/results-cache"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/ruler/ruler-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-ruler
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: ruler
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: ruler
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/ruler"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/templates/store-gateway/store-gateway-servmon.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: release-name-mimir-store-gateway
  namespace: "default"
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: store-gateway
    app.kubernetes.io/part-of: memberlist
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
spec:
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: mimir
      app.kubernetes.io/instance: release-name
      app.kubernetes.io/component: store-gateway
    matchExpressions:
      - key: prometheus.io/service-monitor
        operator: NotIn
        values:
          - "false"
  endpoints:
    - port: http-metrics
      relabelings:
        - sourceLabels: [job]
          replacement: "default/store-gateway"
          targetLabel: job
        - replacement: "release-name"
          targetLabel: cluster
      scheme: http
---
# Source: mimir/charts/mimir/charts/minio/templates/post-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: release-name-minio-post-job
  labels:
    app: minio-post-job
    chart: minio-5.0.14
    release: release-name
    heritage: Helm
  annotations:
    "helm.sh/hook": post-install,post-upgrade
    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
spec:
  template:
    metadata:
      labels:
        app: minio-job
        release: release-name
    spec:
      restartPolicy: OnFailure      
      volumes:
        - name: etc-path
          emptyDir: {}
        - name: tmp
          emptyDir: {}
        - name: minio-configuration
          projected:
            sources:
              - configMap:
                  name: release-name-minio
              - secret:
                  name: release-name-minio
      serviceAccountName: minio-sa
      containers:
        - name: minio-make-bucket
          image: "quay.io/minio/mc:RELEASE.2023-09-29T16-41-22Z"
          imagePullPolicy: IfNotPresent
          command: [ "/bin/sh", "/config/initialize" ]
          env:
            - name: MINIO_ENDPOINT
              value: release-name-minio
            - name: MINIO_PORT
              value: "9000"
          volumeMounts:
            - name: etc-path
              mountPath: /etc/minio/mc
            - name: tmp
              mountPath: /tmp
            - name: minio-configuration
              mountPath: /config
          resources:
            requests:
              memory: 128Mi
        - name: minio-make-user
          image: "quay.io/minio/mc:RELEASE.2023-09-29T16-41-22Z"
          imagePullPolicy: IfNotPresent
          command: [ "/bin/sh", "/config/add-user" ]
          env:
            - name: MINIO_ENDPOINT
              value: release-name-minio
            - name: MINIO_PORT
              value: "9000"
          volumeMounts:
            - name: etc-path
              mountPath: /etc/minio/mc
            - name: tmp
              mountPath: /tmp
            - name: minio-configuration
              mountPath: /config
          resources:
            requests:
              memory: 128Mi
---
# Source: mimir/charts/mimir/templates/smoke-test/smoke-test-job.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: release-name-mimir-smoke-test
  labels:
    helm.sh/chart: mimir-5.2.1
    app.kubernetes.io/name: mimir
    app.kubernetes.io/instance: release-name
    app.kubernetes.io/component: smoke-test
    app.kubernetes.io/version: "2.11.0"
    app.kubernetes.io/managed-by: Helm
  annotations:
    "helm.sh/hook": test
  namespace: "default"
spec:
  backoffLimit: 5
  completions: 1
  parallelism: 1
  selector:
  template:
    metadata:
      labels:
        helm.sh/chart: mimir-5.2.1
        app.kubernetes.io/name: mimir
        app.kubernetes.io/instance: release-name
        app.kubernetes.io/version: "2.11.0"
        app.kubernetes.io/managed-by: Helm
        app.kubernetes.io/component: smoke-test
    spec:
      serviceAccountName: release-name-mimir
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
        seccompProfile:
          type: RuntimeDefault
      initContainers:
        []
      containers:
        - name: smoke-test
          image: "grafana/mimir-continuous-test:2.11.0"
          imagePullPolicy: IfNotPresent
          args:
            - "-tests.smoke-test"
            - "-tests.write-endpoint=http://release-name-mimir-nginx.default.svc:80"
            - "-tests.read-endpoint=http://release-name-mimir-nginx.default.svc:80/prometheus"
            - "-tests.tenant-id="
            - "-tests.write-read-series-test.num-series=1000"
            - "-tests.write-read-series-test.max-query-age=48h"
            - "-server.metrics-port=8080"
          volumeMounts:
          env:
          envFrom:
      restartPolicy: OnFailure
      volumes: