kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml
1
---
2
# yaml-language-server: $schema=https://k8s-schemas.home-operations.com/helm.toolkit.fluxcd.io/helmrelease_v2.json
3
apiVersion: helm.toolkit.fluxcd.io/v2
4
kind: HelmRelease
5
metadata:
6
name: kube-prometheus-stack
7
spec:
8
chartRef:
9
kind: OCIRepository
10
name: kube-prometheus-stack
11
interval: 1h
12
values:
13
cleanPrometheusOperatorObjectNames: true
14
alertmanager:
15
route:
16
main:
17
annotations:
18
gethomepage.dev/enabled: "true"
19
gethomepage.dev/description: Alertmanager for Prometheus alerts
20
gethomepage.dev/group: Observability
21
gethomepage.dev/icon: mdi-alert-octagram
22
gethomepage.dev/name: Alertmanager
23
gethomepage.dev/pod-selector: app.kubernetes.io/name=alertmanager
24
gethomepage.dev/weight: "10"
25
enabled: true
26
hostnames:
27
- alertmanager.janpuc.com
28
parentRefs:
29
- name: envoy-internal
30
namespace: network
31
alertmanagerSpec:
32
alertmanagerConfiguration:
33
name: alertmanager
34
externalUrl: https://alertmanager.janpuc.com
35
resources:
36
requests:
37
cpu: 10m
38
memory: 48Mi
39
limits:
40
memory: 96Mi
41
storage:
42
volumeClaimTemplate:
43
spec:
44
storageClassName: miroir-slow-local
45
resources:
46
requests:
47
storage: 1Gi
48
kubeEtcd:
49
service:
50
selector:
51
component: kube-apiserver
52
kubeProxy:
53
enabled: false
54
prometheus:
55
route:
56
main:
57
annotations:
58
gethomepage.dev/enabled: "true"
59
gethomepage.dev/description: Prometheus monitoring system
60
gethomepage.dev/group: Observability
61
gethomepage.dev/icon: si-prometheus
62
gethomepage.dev/name: Prometheus
63
gethomepage.dev/widget.type: prometheus
64
gethomepage.dev/widget.url: "http://prometheus-operated.observability.svc.cluster.local:9090"
65
gethomepage.dev/pod-selector: app.kubernetes.io/name=prometheus
66
gethomepage.dev/weight: "10"
67
enabled: true
68
hostnames:
69
- prometheus.janpuc.com
70
parentRefs:
71
- name: envoy-internal
72
namespace: network
73
prometheusSpec:
74
externalUrl: https://prometheus.janpuc.com
75
podMonitorSelectorNilUsesHelmValues: false
76
probeSelectorNilUsesHelmValues: false
77
ruleSelectorNilUsesHelmValues: false
78
scrapeConfigSelectorNilUsesHelmValues: false
79
serviceMonitorSelectorNilUsesHelmValues: false
80
retention: 14d
81
retentionSize: 50GB
82
resources:
83
requests:
84
cpu: 300m
85
memory: 2048Mi
86
limits:
87
memory: 5120Mi
88
storageSpec:
89
volumeClaimTemplate:
90
spec:
91
storageClassName: miroir-slow-local
92
resources:
93
requests:
94
storage: 50Gi
95
prometheusOperator:
96
prometheusConfigReloader:
97
resources:
98
requests:
99
cpu: 10m
100
memory: 24Mi
101
limits:
102
memory: 48Mi
103
resources:
104
requests:
105
cpu: 10m
106
memory: 48Mi
107
limits:
108
memory: 96Mi
109
prometheus-node-exporter:
110
fullnameOverride: node-exporter
111
resources:
112
requests:
113
cpu: 10m
114
memory: 64Mi
115
limits:
116
memory: 96Mi
117
extraArgs:
118
- --collector.drm
119
kube-state-metrics:
120
fullnameOverride: kube-state-metrics
121
resources:
122
requests:
123
cpu: 10m
124
memory: 64Mi
125
limits:
126
memory: 256Mi
127
grafana:
128
enabled: false
129
forceDeployDashboards: true
130
operator:
131
dashboardsConfigMapRefEnabled: true
132
folder: observability
133
matchLabels:
134
dashboards: grafana
135
additionalPrometheusRulesMap:
136
amd-gpu-rules:
137
groups:
138
- name: amd-gpu-memory
139
interval: 30s
140
rules:
141
- record: node:amd_gtt_free_bytes
142
expr: node_drm_memory_gtt_size_bytes - node_drm_memory_gtt_used_bytes
143
- record: node:amd_vram_free_bytes
144
expr: node_drm_memory_vram_size_bytes - node_drm_memory_vram_used_bytes
145
- record: node:effective_mem_available_bytes
146
expr: |
147
node_memory_MemAvailable_bytes
148
+ on(instance) node_drm_memory_gtt_size_bytes
149
- on(instance) node_drm_memory_gtt_used_bytes
150
- alert: AmdGttRunningLow
151
expr: |
152
(
153
node:amd_gtt_free_bytes
154
/ on(instance) node_drm_memory_gtt_size_bytes
155
) < 0.20
156
for: 2m
157
labels: { severity: warning }
158
annotations:
159
summary: "AMD iGPU GTT free < 20 % of cap on {{ $labels.instance }}"
160
description: |
161
Only {{ $value | humanizePercentage }} of the configured GTT
162
aperture is free on {{ $labels.instance }} (cap =
163
{{ with query (printf "node_drm_memory_gtt_size_bytes{instance='%s'}" $labels.instance) }}{{ . | first | value | humanize1024 }}B{{ end }}).
164
A Vulkan/ROCm workload is pulling close to its GTT budget;
165
further allocations may fail or, if the kernel has not yet
166
enforced the cap, starve other workloads of memory kubelet
167
cannot see.
168
- alert: AmdGttCritical
169
expr: |
170
(
171
node:amd_gtt_free_bytes
172
/ on(instance) node_drm_memory_gtt_size_bytes
173
) < 0.10
174
for: 1m
175
labels: { severity: critical }
176
annotations:
177
summary: "AMD iGPU GTT free < 10 % of cap on {{ $labels.instance }}"
178
description: |
179
Critical: only {{ $value | humanizePercentage }} of the
180
configured GTT aperture is free on {{ $labels.instance }}.
181
Expect allocation failures or, if a misbehaving workload
182
slipped past the cap, cascading OOMs.
183
- alert: NodeEffectiveMemoryLow
184
expr: node:effective_mem_available_bytes < 2.5 * 1024^3
185
for: 2m
186
labels: { severity: warning }
187
annotations:
188
summary: "Effective memory low on {{ $labels.instance }} ({{ $value | humanize1024 }}B)"
189
description: |
190
cgroup-reclaimable memory + free GTT combined is below 2.5 GiB on
191
{{ $labels.instance }}. Consider draining the GTT workload.
192
- alert: AmdDrmCollectorMissing
193
expr: absent(node_drm_memory_gtt_size_bytes)
194
for: 5m
195
labels: { severity: warning }
196
annotations:
197
summary: "node_exporter drm collector stopped reporting"
198
description: "Re-apply node-exporter extraArgs in kube-prometheus-stack HelmRelease."
199
- alert: AmdGttCapMismatch
200
expr: |
201
node_drm_memory_gtt_size_bytes != 2147483648
202
and node_drm_memory_gtt_size_bytes != 3221225472
203
for: 10m
204
labels: { severity: warning }
205
annotations:
206
summary: "GTT cap on {{ $labels.instance }} doesn't match any configured value"
207
description: |
208
node_drm_memory_gtt_size_bytes = {{ $value | humanize1024 }}B
209
on {{ $labels.instance }}, which is neither the media
210
cap (2 GiB) nor the AI cap (3 GiB). The node has likely
211
not been rebooted since the gttsize= module option was
212
added to its Talos config, or Talos did not honour the
213
option. Reboot the node to pick up the new module
214
parameters.
215
dockerhub-rules:
216
groups:
217
- name: dockerhub
218
rules:
219
- alert: DockerhubRateLimitRisk
220
annotations:
221
summary: There are {{ $value }} containers pulling from Dockerhub. This may lead to rate limiting.
222
expr: count(time() - container_last_seen{image=~"(docker.io).*",container!=""} < 30) > 100
223
labels:
224
severity: critical
225
oom-rules:
226
groups:
227
- name: oom
228
rules:
229
- alert: OomKilled
230
annotations:
231
summary: Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.
232
expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
233
labels:
234
severity: critical
235
zfs-rules:
236
groups:
237
- name: zfs
238
rules:
239
- alert: ZfsUnexpectedPoolState
240
annotations:
241
summary: ZFS pool {{$labels.zpool}} on {{$labels.instance}} is in a unexpected state {{$labels.state}}
242
expr: node_zfs_zpool_state{state!="online"} > 0
243
labels:
244
severity: critical