Files

kubernetes/apps/miroir-system/miroir-medic/app/configmap.yaml

1 ---

2 apiVersion: v1

3 kind: ConfigMap

4 metadata:

5 name: miroir-medic

6 data:

7 medic.sh: |

8 #!/bin/sh

9 set -u

11 PERSISTENT_STATE=/var/lib/miroir-medic

12 BOOT_STATE=/run/miroir-medic

13 FATAL_ASSERTION_PATTERN='ASSERTION.*FAILED in (put_ldev|drbd_al_begin_io_fastpath)'

14 PEER_IPS="${PEER_IPS:-192.168.69.20 192.168.69.21 192.168.69.22}"

15 NODE="${NODE_NAME:-unknown}"

16 SELF_IP="${HOST_IP:?HOST_IP is required}"

17 SETTLE_UPTIME_SECONDS=900

18 KUBELET_CONSECUTIVE_FAILS_BEFORE_RESET=20

19 STRANDED_TASKS_MIN=3

20 STRANDED_COMMS_MIN=2

21 STRANDED_POLLS_BEFORE_RESET=30

22 PEERS_DOWN_OVERRIDE_SECONDS=1800

23 MIN_SECONDS_BETWEEN_RESETS=1800

24 MAX_RESETS_PER_6H=3

25 MAX_RESETS_PER_24H=5

26 STANDDOWN_RENOTIFY_SECONDS=21600

27 POLL_SECONDS=30

29 mkdir -p "$PERSISTENT_STATE" "$BOOT_STATE"

31 log() { echo "$(date -u +%FT%TZ) $*"; }

33 epoch_now() { date +%s; }

35 uptime_seconds() { cut -d. -f1 /proc/uptime; }

37 notify() {

38 [ -n "${PUSHOVER_TOKEN:-}" ] && [ -n "${PUSHOVER_USER:-}" ] || return 0

39 wget -T 8 -q -O /dev/null --post-data \

40 "token=$PUSHOVER_TOKEN&user=$PUSHOVER_USER&priority=1&title=miroir-medic $NODE&message=$1" \

41 https://api.pushover.net/1/messages.json 2>/dev/null || log "notify failed: $1"

42 }

44 resets_within() {

45 cutoff=$(( $(epoch_now) - $1 ))

46 [ -f "$PERSISTENT_STATE/resets" ] || { echo 0; return; }

47 awk -v c="$cutoff" '$1 >= c' "$PERSISTENT_STATE/resets" | wc -l

48 }

50 seconds_since_last_reset() {

51 last=$(tail -1 "$PERSISTENT_STATE/resets" 2>/dev/null)

52 echo $(( $(epoch_now) - ${last:-0} ))

53 }

55 latch_wedge_if_asserting() {

56 [ -f "$BOOT_STATE/wedge" ] && return 0

57 if dmesg 2>/dev/null | grep -qE "$FATAL_ASSERTION_PATTERN"; then

58 log "fatal DRBD assertion sighted in kernel log; latching wedge"

59 dmesg 2>/dev/null | grep -E "$FATAL_ASSERTION_PATTERN" | tail -3

60 touch "$BOOT_STATE/wedge"

61 fi

62 return 0

63 }

65 wedged() { [ -f "$BOOT_STATE/wedge" ]; }

67 scan_d_tasks() {

68 : > "$BOOT_STATE/dtasks.now"

69 for stat_file in /proc/[0-9]*/stat; do

70 read -r stat_line 2>/dev/null < "$stat_file" || continue

71 after_comm="${stat_line##*) }"

72 task_state="${after_comm%% *}"

73 [ "$task_state" = "D" ] || continue

74 task_pid="${stat_line%% *}"

75 [ -s "/proc/$task_pid/cmdline" ] || continue

76 task_comm="${stat_line#*(}"

77 task_comm="${task_comm%)*}"

78 echo "$task_pid ($task_comm)" >> "$BOOT_STATE/dtasks.now"

79 done

80 }

82 stranded_storage_check() {

83 scan_d_tasks

84 if [ ! -f "$BOOT_STATE/dtasks.baseline" ]; then

85 cp "$BOOT_STATE/dtasks.now" "$BOOT_STATE/dtasks.baseline"

86 return 1

87 fi

88 : > "$BOOT_STATE/dtasks.streak.new"

89 : > "$BOOT_STATE/dtasks.stuck"

90 stuck_count=0

91 while IFS= read -r task; do

92 grep -Fxq "$task" "$BOOT_STATE/dtasks.baseline" && continue

93 prev_line=$(grep -F "|$task" "$BOOT_STATE/dtasks.streak" 2>/dev/null | head -1)

94 prev_count="${prev_line%%|*}"

95 streak=$(( ${prev_count:-0} + 1 ))

96 echo "$streak|$task" >> "$BOOT_STATE/dtasks.streak.new"

97 if [ "$streak" -ge "$STRANDED_POLLS_BEFORE_RESET" ]; then

98 stuck_count=$(( stuck_count + 1 ))

99 echo "${task#* }" >> "$BOOT_STATE/dtasks.stuck"

100 fi

101 done < "$BOOT_STATE/dtasks.now"

102 mv "$BOOT_STATE/dtasks.streak.new" "$BOOT_STATE/dtasks.streak"

103 distinct_comms=$(sort -u "$BOOT_STATE/dtasks.stuck" | wc -l)

104 if [ "$stuck_count" -ge "$STRANDED_TASKS_MIN" ] && [ "$distinct_comms" -ge "$STRANDED_COMMS_MIN" ]; then

105 stuck_tasks=$(sort -rn "$BOOT_STATE/dtasks.streak" | head -5 | cut -d'|' -f2 | tr '\n' ' ')

106

        log "storage stranded: $stuck_count userspace tasks ($distinct_comms distinct commands) in uninterruptible sleep beyond the window: $stuck_tasks"

107 return 0

108 fi

109 return 1

110 }

111

112 kubelet_probe_ok() { wget -T 5 -q -O /dev/null http://127.0.0.1:10248/healthz 2>/dev/null; }

113

114 kubelet_unhealthy_too_long() {

115 if kubelet_probe_ok; then

116 rm -f "$BOOT_STATE/kubelet_fails"

117 return 1

118 fi

119 fails=$(( $(cat "$BOOT_STATE/kubelet_fails" 2>/dev/null || echo 0) + 1 ))

120 echo "$fails" > "$BOOT_STATE/kubelet_fails"

121 [ "$fails" = "$KUBELET_CONSECUTIVE_FAILS_BEFORE_RESET" ] && log "kubelet healthz failing ($fails consecutive)"

122 [ "$fails" -ge "$KUBELET_CONSECUTIVE_FAILS_BEFORE_RESET" ]

123 }

124

125 peer_serving_apiserver() {

126 wget -T 4 -q -O /dev/null --no-check-certificate "https://$1:6443/healthz" 2>/dev/null

127 }

128

129 peers_gate() {

130 any_down=0

131 for peer in $PEER_IPS; do

132 [ "$peer" = "$SELF_IP" ] && continue

133 peer_serving_apiserver "$peer" || any_down=1

134 done

135 if [ "$any_down" = 0 ]; then

136 rm -f "$BOOT_STATE/peers_down_since"

137 return 0

138 fi

139 [ -f "$BOOT_STATE/peers_down_since" ] || epoch_now > "$BOOT_STATE/peers_down_since"

140 down_since=$(cat "$BOOT_STATE/peers_down_since")

141 if [ "$1" = allow_override ] && [ $(( $(epoch_now) - down_since )) -ge "$PEERS_DOWN_OVERRIDE_SECONDS" ]; then

142 log "peer apiserver down ${PEERS_DOWN_OVERRIDE_SECONDS}s+ while self wedged; overriding quorum hold"

143 return 0

144 fi

145 return 1

146 }

147

148 stagger_by_node() {

149 case "$SELF_IP" in

150 *.20) sleep 0 ;;

151 *.21) sleep 90 ;;

152 *.22) sleep 180 ;;

153 *) sleep 45 ;;

154 esac

155 }

156

157 attempt_reset() {

158 reason="$1"

159 used_6h=$(resets_within 21600)

160 used_24h=$(resets_within 86400)

161 if [ "$(seconds_since_last_reset)" -lt "$MIN_SECONDS_BETWEEN_RESETS" ] \

162 || [ "$used_6h" -ge "$MAX_RESETS_PER_6H" ] \

163 || [ "$used_24h" -ge "$MAX_RESETS_PER_24H" ]; then

164 last_warned=$(cat "$PERSISTENT_STATE/standdown_warned" 2>/dev/null || echo 0)

165 if [ $(( $(epoch_now) - last_warned )) -ge "$STANDDOWN_RENOTIFY_SECONDS" ]; then

166 epoch_now > "$PERSISTENT_STATE/standdown_warned"

167 log "reset budget exhausted ($reason, 6h=$used_6h 24h=$used_24h); standing down"

168

          notify "$reason persists but reset budget is exhausted (6h=$used_6h 24h=$used_24h); node left fenced, needs manual attention"

169 fi

170 return 0

171 fi

172 stagger_by_node

173 if [ "$reason" = kubelet ] && kubelet_probe_ok; then

174 log "kubelet recovered during stagger; aborting reset"

175 rm -f "$BOOT_STATE/kubelet_fails"

176 return 0

177 fi

178 if [ "$reason" = stranded ] && ! stranded_storage_check; then

179 log "stranded tasks drained during stagger; aborting reset"

180 return 0

181 fi

182 epoch_now >> "$PERSISTENT_STATE/resets"

183 echo "$reason $(date -u +%FT%TZ)" > "$PERSISTENT_STATE/pending"

184 log "hard reset now: $reason (reset $((used_24h + 1)) of $MAX_RESETS_PER_24H per 24h)"

185 notify "$reason detected; hard-resetting this node (reset $((used_24h + 1)) of $MAX_RESETS_PER_24H per 24h)"

186 sleep 2

187 echo s > /proc/sysrq-trigger 2>/dev/null

188 sleep 8

189 echo b > /proc/sysrq-trigger 2>/dev/null || notify "sysrq reset write FAILED; node still wedged"

190 sleep 300

191 }

192

193 if [ -f "$PERSISTENT_STATE/pending" ]; then

194 if [ "$(uptime_seconds)" -lt 3600 ]; then

195 notify "node is back after hard reset ($(cat "$PERSISTENT_STATE/pending")); uptime $(uptime_seconds)s"

196 fi

197 rm -f "$PERSISTENT_STATE/pending"

198 fi

199

200 log "miroir-medic watching on $NODE ($SELF_IP); resets used 24h=$(resets_within 86400)"

201

202 while :; do

203 sleep "$POLL_SECONDS"

204 [ "$(uptime_seconds)" -ge "$SETTLE_UPTIME_SECONDS" ] || continue

205 latch_wedge_if_asserting

206 if wedged; then

207 peers_gate allow_override && attempt_reset wedge

208 continue

209 fi

210 if stranded_storage_check; then

211 peers_gate no_override && attempt_reset stranded

212 continue

213 fi

214 if kubelet_unhealthy_too_long; then

215 peers_gate no_override && attempt_reset kubelet

216 fi

217 done

Reset gates: settle, peer quorum, stagger, budget — in that order

invariant ·ok L17

The script's safety comes from four gates that must stay in this order; each exists because of a specific way an automatic hard reset can make things worse on a 3-node all-control-plane cluster.

Show the rest

- SETTLE_UPTIME_SECONDS (15 min): resync after a boot is exactly the put_ldev-heavy transfer that re-triggers LINBIT/drbd#137 — k8s-0 re-latched 6 minutes after a manual reboot on 2026-09-01. The settle window also bounds the tightest possible reboot loop. Cross-boot false latches are impossible (the kmsg ring and /run flags both clear on boot), so no extra guard is needed there. - peers_gate: self-reset drops etcd to 2/3 only if both other members are up, so both peers must answer on :6443 first (anonymous /healthz is enabled cluster-wide in KubeAuthenticationConfig). The allow_override branch exists because a wedge plus a 30-minutes-dead peer means the cluster is already down — rolling staggered resets are then the only path back, and that is precisely how the 2026-09-01 outage was resolved by hand. The kubelet reason never overrides: a kubelet-dead node with a dead peer must hold, since its own reset would take etcd from 2/3 to 1/3. - stagger_by_node (0/90/180s by host IP): two nodes latching in the same window must not reset simultaneously; whoever fires first closes the other's peers_gate. - budgets (30 min spacing, 3/6h, 5/24h in /var/lib, which survives reboots): if resets are not curing the wedge, looping power cycles only multiply resync storms. The end state after an exhausted budget is deliberate: node stays fenced by miroir's taint, cluster degrades to two nodes, Pushover says so every 6h.

kubelet_probe_ok is re-checked after the stagger for the kubelet reason only; the wedge latch is permanent for the boot by design (matching miroir's Wedge.Latch semantics — the kernel state cannot recover).

2026-09-0101M1DYCFC7E1ZTA9ATZXV45SG2

D-state detector covers the assertion-less wedge that blinded v1

why ·ok L67

On 2026-09-01 ~16:17Z k8s-2's storage died the quiet way: miroir's breaker latched on stranded children (stuck lvm/mount commands), no kernel assertion was ever printed, kubelet stayed healthy — and the kmsg-only medic v1 sat correctly silent while a human had to force-reboot two nodes. Worse, the StorageWedged condition un-latched itself four minutes later when the agent restarted and the kmsg replay was clean, so even condition-watching would have missed it; the agent's view can lie across restarts, the kernel's cannot.

Show the rest

Sustained uninterruptible sleep is the honest node-local signal for that class: every I/O against a dead device parks its task in D. The thresholds encode the difference between stuck and slow — a real wedge strands many tasks at once and forever, while heavy-but-healthy I/O (a kopia mover pushing to the NAS over NFS) can legitimately hold one or two tasks in D for minutes. Hence ≥3 distinct non-baseline tasks, each D across 30 consecutive 30s polls (15 min), with the streak reset the moment a task leaves D.

The boot-time baseline set (first scan after the settle window) is excluded forever: some kernel threads park in D legitimately, and whatever was already stuck before the medic could judge it predates its ability to reason about it. Unlike the assertion latch this check is evaluated live, never latched — D-state can drain, and attempt_reset re-verifies after the stagger so a recovered node is not shot. Parsing uses /proc/<pid>/stat with the state taken from after the last ") " so comms containing parentheses or spaces cannot shift the field; requires hostPID on the DaemonSet, or the medic sees only its own container's processes.

2026-09-0101M1F0JX8NGJY9F5WWQ8W09WS2

Kernel threads must never count as stranded — this line fired a node

gotcha ·ok L75

The first deployment of this detector had no kernel-thread exclusion and hard-reset healthy k8s-1 at 2026-09-01T17:56Z, 21 minutes after rollout: DRBD's own resync receiver threads camp in uninterruptible sleep for the whole duration of a large transfer, three of them appeared after the boot-time baseline (the resyncs started later), and 30 polls later they satisfied every threshold. The reset destroyed the resync it was measuring and regressed ~40 volumes — the exact storm the budgets exist to cap.

Show the rest

An empty /proc/<pid>/cmdline is the kernel-thread test (also race-safe: a PID that exits mid-scan reads as empty and is skipped). Userspace-only D-state is the honest wedge signal — it is also what miroir's own breaker counts (stuck lvm/mount/kopia processes). STRANDED_COMMS_MIN adds the second discriminator: a real wedge strands *different* programs at once, while legitimately slow I/O (a kopia mover pushing multi-GB blobs to the NAS) holds one command in D at a time, so three stuck tasks must span at least two distinct comms before the medic may fire.

2026-09-0101M1F21H8VGAQ8MERHKBKNASHW