Files

kubernetes/apps/ai/hermes/app/configmap.yaml

1 apiVersion: v1

2 kind: ConfigMap

3 metadata:

4 name: hermes-configmap

5 data:

6 config.yaml: |-

7 custom_providers:

8 - name: litellm

9 base_url: http://litellm.ai:4000/v1

10 key_env: LITELLM_API_KEY

11 models:

12 minimax/MiniMax-M3:

13 context_length: 1000000

14 opencode-go/minimax-m3:

15 context_length: 1000000

16 opencode-go/glm-5.2:

17 context_length: 1000000

18 opencode-go/glm-5.3:

19 context_length: 1000000

20 opencode-go/deepseek-v4-flash:

21 context_length: 1000000

22 opencode-go/deepseek-v4-flash-vision-exp:

23 context_length: 1000000

24 opencode-go/mimo-v2.5:

25 context_length: 1000000

26 opencode-go/qwen3.6-plus:

27 context_length: 1000000

28 opencode-go/kimi-k3:

29 context_length: 1048576

30 opencode-go/kimi-k2.7-code:

31 context_length: 262144

32 opencode-go/deepseek-v4-pro:

33 context_length: 1000000

34 opencode-go/qwen3.7-max:

35 context_length: 1000000

36 opencode-go/qwen3.7-plus:

37 context_length: 1000000

38 opencode-go/qwen3.8-max:

39 context_length: 1000000

40 opencode-go/mimo-v2.5-pro:

41 context_length: 1048576

42 opencode-go/grok-4.5:

43 context_length: 500000

44 opencode-go/minimax-m2.7:

45 context_length: 204800

46 opencode-go/gpt-5.6-luna:

47 context_length: 1050000

48 opencode-go/hy3:

49 context_length: 256000

50 opencode-go/ox-alpha-free:

51 context_length: 1000000

52 model:

53 default: minimax/MiniMax-M3

54 provider: custom:litellm

55 context_length: 1000000

56 model_aliases:

57 qwen:

58 model: opencode-go/qwen3.6-plus

59 provider: custom:litellm

60 minimax:

61 model: minimax/MiniMax-M3

62 provider: custom:litellm

63 glm:

64 model: opencode-go/glm-5.2

65 provider: custom:litellm

66 alpha:

67 model: opencode-go/ox-alpha-free

68 provider: custom:litellm

69 luna:

70 model: opencode-go/gpt-5.6-luna

71 provider: custom:litellm

72 hy:

73 model: opencode-go/hy3

74 provider: custom:litellm

75 flash-vision:

76 model: opencode-go/deepseek-v4-flash-vision-exp

77 provider: custom:litellm

78 flash:

79 model: opencode-go/deepseek-v4-flash-vision-exp

80 provider: custom:litellm

81 mimo:

82 model: opencode-go/mimo-v2.5

83 provider: custom:litellm

84 k3:

85 model: opencode-go/kimi-k3

86 provider: custom:litellm

87 kimi-code:

88 model: opencode-go/kimi-k2.7-code

89 provider: custom:litellm

90 deepseek:

91 model: opencode-go/deepseek-v4-flash-vision-exp

92 provider: custom:litellm

93 grok:

94 model: opencode-go/grok-4.5

95 provider: custom:litellm

96 qwen-max:

97 model: opencode-go/qwen3.8-max

98 provider: custom:litellm

99 fallback_providers:

100 - provider: custom:litellm

101 model: opencode-go/minimax-m3

102 - provider: custom:litellm

103 model: opencode-go/glm-5.2

104 - provider: custom:litellm

105 model: opencode-go/deepseek-v4-flash-vision-exp

106 auxiliary:

107 approval:

108 provider: custom:litellm

109 model: minimax/MiniMax-M3

110 timeout: 60

111 compression:

112 provider: custom:litellm

113 model: minimax/MiniMax-M3

114 web_extract:

115 provider: custom:litellm

116 model: minimax/MiniMax-M3

117 session_search:

118 provider: custom:litellm

119 model: opencode-go/hy3

120 vision:

121 provider: custom:litellm

122 model: minimax/MiniMax-M3

123 goal_judge:

124 provider: custom:litellm

125 model: opencode-go/deepseek-v4-flash-vision-exp

126 title_generation:

127 provider: custom:litellm

128 model: opencode-go/hy3

129 profile_describer:

130 provider: custom:litellm

131 model: opencode-go/hy3

132 triage_specifier:

133 provider: custom:litellm

134 model: opencode-go/hy3

135 kanban_decomposer:

136 provider: custom:litellm

137 model: opencode-go/deepseek-v4-flash-vision-exp

138 curator:

139 provider: custom:litellm

140 model: opencode-go/deepseek-v4-flash-vision-exp

141 timeout: 600

142 memory:

143 provider: memini

144 plugins:

145 enabled:

146 - memini

147 terminal:

148 backend: local

149 cwd: /opt/data/workspace

150 timeout: 900

151 lifetime_seconds: 1800

152 persistent_shell: true

153 compression:

154 enabled: true

155 protect_last_n: 20

156 threshold: 0.5

157 streaming:

158 enabled: false

159 approvals:

160 mode: smart

161 timeout: 60

162 cron_mode: approve

163 deny:

164 - "*talosctl*reset*"

165 - "*talosctl*wipe*"

166 - "*talosctl*upgrade*"

167 - "git push*--force*"

168 - "git push*-f *"

169 - "git reset --hard*"

170 - "git clean*-*f*"

171 - "git filter-branch*"

172 - "*flux uninstall*"

173 - "*helm uninstall*"

174 - "*helm delete*"

175 - "*kubectl delete*--all*"

176 - "*kubectl delete ns*"

177 - "*kubectl delete namespace*"

178 - "*kubectl delete pvc*"

179 - "*kubectl delete pv *"

180 - "*mkfs*"

181 - "*rm -rf /*"

182 - "*dd if=*of=/dev/*"

183 - "*wipefs*"

184 - "*curl*|*sh*"

185 - "*wget*|*sh*"

186 hooks:

187 pre_tool_call:

188 - command: /usr/bin/python3 /opt/data/hook-scripts/home-ops-guard.py

189 matcher: terminal

190 timeout: 10

191 hooks_auto_accept: true

192 security:

193 redact_secrets: true

194 website_blocklist:

195 enabled: true

196 domains:

197 - "169.254.169.254"

198 - "metadata.google.internal"

199 privacy:

200 redact_pii: true

201 agent:

202 max_turns: 150

203 gateway_timeout: 1800

204 gateway_timeout_warning: 900

205 restart_drain_timeout: 60

206 api_max_retries: 3

207 reasoning_effort: medium

208 delegation:

209 model: opencode-go/deepseek-v4-flash-vision-exp

210 max_concurrent_children: 8

211 max_spawn_depth: 2

212 moa:

213 default_preset: council

214 presets:

215 council:

216 enabled: true

217 reference_models:

218 - provider: custom:litellm

219 model: opencode-go/kimi-k2.6

220 - provider: custom:litellm

221 model: opencode-go/glm-5.2

222 - provider: custom:litellm

223 model: opencode-go/deepseek-v4-flash-vision-exp

224 - provider: custom:litellm

225 model: opencode-go/qwen3.7-max

226 aggregator:

227 provider: custom:litellm

228 model: minimax/MiniMax-M3

229 reference_temperature: 0.6

230 aggregator_temperature: 0.4

231 fanout: user_turn

232 reference_max_tokens: 600

233 max_tokens: 16384

234 quick:

235 enabled: true

236 reference_models:

237 - provider: custom:litellm

238 model: opencode-go/hy3

239 - provider: custom:litellm

240 model: opencode-go/qwen3.7-plus

241 aggregator:

242 provider: custom:litellm

243 model: opencode-go/glm-5.2

244 reference_temperature: 0.7

245 aggregator_temperature: 0.3

246 reference_max_tokens: 400

247 max_tokens: 8192

248 platforms:

249 matrix:

250 enabled: false

251 matrix:

252 require_mention: false

253 auto_thread: false

254 dm_mention_threads: false

255 display:

256 platforms:

257 matrix:

258 tool_progress: "off"

259 mcp_servers:

260 kubectl:

261 url: http://litellm.ai:4000/mcp/kubectl/

262 headers:

263 Authorization: "Bearer $${LITELLM_API_KEY}"

264 tools:

265 include:

266 - kubectl-get_pods

267 - kubectl-get_nodes

268 - kubectl-get_nodes_summary

269 - kubectl-get_node_metrics

270 - kubectl-get_pod_metrics

271 - kubectl-get_deployments

272 - kubectl-get_statefulsets

273 - kubectl-get_daemonsets

274 - kubectl-get_replicasets

275 - kubectl-get_services

276 - kubectl-get_endpoints

277 - kubectl-get_ingress

278 - kubectl-get_configmaps

279 - kubectl-get_secrets

280 - kubectl-get_namespaces

281 - kubectl-get_events

282 - kubectl-get_pod_events

283 - kubectl-get_pod_conditions

284 - kubectl-get_pvcs

285 - kubectl-get_persistent_volumes

286 - kubectl-get_storage_classes

287 - kubectl-get_jobs

288 - kubectl-get_hpa

289 - kubectl-get_pdb

290 - kubectl-get_crds

291 - kubectl-get_api_resources

292 - kubectl-get_cluster_info

293 - kubectl-get_cluster_version

294 - kubectl-get_logs

295 - kubectl-get_previous_logs

296 - kubectl-get_resource_quotas

297 - kubectl-get_service_accounts

298 - kubectl-get_rbac_roles

299 - kubectl-get_cluster_roles

300 - kubectl-health_check

301 - kubectl-check_pod_health

302 - kubectl-diagnose_pod_crash

303 - kubectl-check_dns_resolution

304 - kubectl-diagnose_network_connectivity

305 - kubectl-detect_pending_pods

306 - kubectl-get_evicted_pods

307 - kubectl-trace_service_chain

308 - kubectl-analyze_pod_security

309 - kubectl-analyze_network_policies

310 - kubectl-audit_rbac_permissions

311 - kubectl-check_crd_exists

312 - kubectl-get_resource_usage

313 - kubectl-get_resource_trends

314 - kubectl-get_resource_recommendations

315 - kubectl-get_overprovisioned_resources

316 - kubectl-get_idle_resources

317 - kubectl-optimize_resource_requests

318 - kubectl-node_top_tool

319 - kubectl-node_stats_summary_tool

320 - kubectl-node_logs_tool

321 - kubectl-node_management

322 - kubectl-exec_in_pod

323 - kubectl-kubectl_generic

324 - kubectl-kubectl_describe

325 - kubectl-kubectl_explain

326 - kubectl-kubectl_apply

327 - kubectl-kubectl_patch

328 - kubectl-kubectl_rollout

329 - kubectl-kubectl_cp

330 - kubectl-port_forward

331 - kubectl-delete_resource

332 - kubectl-restart_deployment

333 - kubectl-scale_deployment

334 - kubectl-cleanup_pods

335 - kubectl-wait_for_condition

336 - kubectl-label_resource

337 - kubectl-annotate_resource

338 - kubectl-cilium_status_tool

339 - kubectl-cilium_endpoints_list_tool

340 - kubectl-cilium_nodes_list_tool

341 - kubectl-cilium_policies_list_tool

342 - kubectl-cilium_policy_get_tool

343 - kubectl-hubble_flows_query_tool

344 - kubectl-certs_list_tool

345 - kubectl-certs_get_tool

346 - kubectl-certs_status_explain_tool

347 - kubectl-certs_challenges_list_tool

348 - kubectl-certs_requests_list_tool

349 - kubectl-certs_issuers_list_tool

350 - kubectl-certs_renew_tool

351 - kubectl-gitops_detect_engine_tool

352 - kubectl-gitops_apps_list_tool

353 - kubectl-gitops_app_get_tool

354 - kubectl-gitops_app_status_tool

355 - kubectl-gitops_app_sync_tool

356 - kubectl-gitops_sources_list_tool

357 - kubectl-gitops_source_get_tool

358 - kubectl-get_current_context

359 - kubectl-get_server_config_status

360 - kubectl-compare_namespaces

361 ha:

362 url: http://litellm.ai:4000/mcp/ha/

363 headers:

364 Authorization: "Bearer $${LITELLM_API_KEY}"

365 tools:

366 include:

367 - ha-ha_get_state

368 - ha-ha_search

369 - ha-ha_call_service

370 - ha-ha_get_history

371 - ha-ha_get_entity

372 - ha-ha_get_overview

373 - ha-ha_list_floors_areas

374 - ha-ha_list_services

375 - ha-ha_eval_template

376 - ha-ha_get_todo

377 - ha-ha_set_todo_item

378 - ha-ha_config_get_calendar_events

379 - ha-ha_get_logs

380 - ha-ha_bulk_control

381 - ha-ha_get_device

382 github:

383 url: http://litellm.ai:4000/mcp/github/

384 headers:

385 Authorization: "Bearer $${LITELLM_API_KEY}"

386 prometheus:

387 url: http://litellm.ai:4000/mcp/prometheus/

388 headers:

389 Authorization: "Bearer $${LITELLM_API_KEY}"

390 victoria_logs:

391 url: http://litellm.ai:4000/mcp/victoria_logs/

392 headers:

393 Authorization: "Bearer $${LITELLM_API_KEY}"

394 context7:

395 url: http://litellm.ai:4000/mcp/context7/

396 headers:

397 Authorization: "Bearer $${LITELLM_API_KEY}"

398 koment:

399 url: http://koment.selfhosted.svc.cluster.local:8080/mcp

400 headers:

401 Authorization: "Bearer $${KOMENT_API_KEY}"

402 web:

403 search_backend: searxng

404 env: |-

405 LITELLM_API_KEY={{ .LITELLM_MASTER_KEY }}

406 OPENAI_API_KEY={{ .LITELLM_MASTER_KEY }}

407 OPENAI_BASE_URL=http://litellm.ai:4000/v1

408 SUMMARY_LLM_API_KEY={{ .LITELLM_MASTER_KEY }}

409 API_SERVER_KEY={{ .API_SERVER_KEY }}

410 GATEWAY_ALLOWED_USERS={{ .GATEWAY_ALLOWED_USERS }}

411 KOMENT_API_KEY={{ .KOMENT_HERMES_TOKEN }}

412 matrix-post.py: |-

413 """Post a message into a named Hermes Matrix room.

414

415 Usage:

416 matrix-post <room-alias> [text ...] (text, or stdin if omitted)

417 matrix-post digest --html < body.html (send pre-rendered HTML)

418 matrix-post notify --text "..." (m.text instead of m.notice)

419

420 Room alias resolves to the MATRIX_ROOM_<ALIAS> environment variable.

421 Exits non-zero with a readable message on any failure so a cron job that

422 cannot deliver fails loudly instead of silently succeeding.

423 """

424 import json

425 import os

426 import sys

427 import time

428 import urllib.error

429 import urllib.request

430

431 ALIASES = ("cluster", "assistant", "home", "notify",

432 "alerts", "digest", "inbox", "dm")

433

434

435 def die(msg, code=1):

436 sys.stderr.write("matrix-post: %s\n" % msg)

437 sys.exit(code)

438

439

440 def main():

441 args = [a for a in sys.argv[1:]]

442 if not args or args[0] in ("-h", "--help"):

443 die("usage: matrix-post <%s> [--html] [--text] [message]"

444 % "|".join(ALIASES), 0 if args else 2)

445

446 alias = args.pop(0).lower()

447 if alias not in ALIASES:

448 die("unknown room %r (expected one of: %s)" % (alias, ", ".join(ALIASES)), 2)

449

450 as_html = False

451 msgtype = "m.notice"

452 rest = []

453 for a in args:

454 if a == "--html":

455 as_html = True

456 elif a == "--text":

457 msgtype = "m.text"

458 elif a == "--notice":

459 msgtype = "m.notice"

460 else:

461 rest.append(a)

462

463 body = " ".join(rest) if rest else sys.stdin.read()

464 body = body.strip()

465 if not body:

466 die("refusing to send an empty message", 2)

467

468 hs = os.environ.get("MATRIX_HOMESERVER", "").rstrip("/")

469 tok = os.environ.get("MATRIX_ACCESS_TOKEN", "")

470 room = os.environ.get("MATRIX_ROOM_" + alias.upper(), "")

471 if not hs or not tok:

472 die("MATRIX_HOMESERVER / MATRIX_ACCESS_TOKEN not set in this environment")

473 if not room:

474 die("MATRIX_ROOM_%s is not set" % alias.upper())

475

476 content = {"msgtype": msgtype, "body": body}

477 if as_html:

478 content["format"] = "org.matrix.custom.html"

479 content["formatted_body"] = body

480

481 txn = "hermes-%d" % int(time.time() * 1000)

482 url = "%s/_matrix/client/v3/rooms/%s/send/m.room.message/%s" % (hs, room, txn)

483 req = urllib.request.Request(

484 url, data=json.dumps(content).encode(), method="PUT",

485 headers={"Authorization": "Bearer " + tok,

486 "Content-Type": "application/json"},

487 )

488 try:

489 with urllib.request.urlopen(req, timeout=30) as r:

490 print(json.loads(r.read().decode()).get("event_id", ""))

491 except urllib.error.HTTPError as e:

492 die("HTTP %s posting to %s: %s" % (e.code, alias, e.read()[:300].decode("utf-8", "replace")))

493 except Exception as e:

494 die("%s posting to %s" % (type(e).__name__, alias))

495

496

497 main()

498

499 weekly-digest.py: |-

500 """Emit the homelab weekly digest as HTML on stdout.

501

502 Deterministic and cheap on purpose: this is a factual rollup, so it runs as a

503 plain script rather than an agent turn. Pipe it into matrix-post:

504

505 weekly-digest | matrix-post digest --html

506

507 Every section degrades to a one-line note instead of raising, so a missing RBAC

508 grant or a CRD that is not installed never costs the whole digest.

509 """

510 import datetime

511 import os

512 import html

513 import json

514 import subprocess

515 import sys

516

517 KUBECTL = "kubectl"

518 NOW = datetime.datetime.now(datetime.timezone.utc)

519 OUT = []

520

521

522 def kc(args, timeout=60):

523 """Run kubectl and return parsed JSON, or None on any failure."""

524 try:

525 p = subprocess.run(

526 [KUBECTL] + args + ["-o", "json"],

527 capture_output=True, text=True, timeout=timeout,

528 )

529 if p.returncode != 0:

530 return None

531 return json.loads(p.stdout)

532 except Exception:

533 return None

534

535

536 def age_days(ts):

537 if not ts:

538 return None

539 try:

540 t = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00"))

541 return (NOW - t).total_seconds() / 86400.0

542 except Exception:

543 return None

544

545

546 def h(s):

547 return html.escape(str(s))

548

549

550 def section(title, lines, empty="nothing to report"):

551 OUT.append("%s " % h(title))

552 if not lines:

553 OUT.append("  %s " % h(empty))

554 return

555 for ln in lines:

556 OUT.append("  %s " % ln)

557 OUT.append(" ")

558

559

560 def ready_cond(obj):

561 for c in (obj.get("status", {}) or {}).get("conditions", []) or []:

562 if c.get("type") == "Ready":

563 return c.get("status"), c.get("reason", ""), c.get("message", "")

564 return None, "", ""

565

566

567 def flux():

568 bad = []

569 total = 0

570 for kind, res in (("Kustomization", "kustomizations"), ("HelmRelease", "helmreleases")):

571 d = kc(["get", res, "-A"])

572 if d is None:

573 bad.append("%s: not readable" % kind)

574 continue

575 for it in d.get("items", []):

576 total += 1

577 st, reason, _ = ready_cond(it)

578 if st != "True":

579 bad.append("%s <code>%s/%s</code> — %s" % (

580 kind, h(it["metadata"]["namespace"]),

581 h(it["metadata"]["name"]), h(reason or st)))

582 section("Flux (%d objects)" % total, bad, "all reconciled")

583

584

585 def storage():

586 d = kc(["get", "miroirvolumes", "-A"])

587 if d is None:

588 section("Storage", ["miroirvolumes not readable"])

589 return

590 phases = {}

591 problems = []

592 for it in d.get("items", []):

593 st = it.get("status", {}) or {}

594 ph = st.get("phase", "?")

595 phases[ph] = phases.get(ph, 0) + 1

596 rr = str(st.get("readyReplicas", ""))

597 if rr.startswith("0/"):

598 problems.append("<code>%s</code> has NO ready replica (%s)"

599 % (h(it["metadata"]["name"][:26]), h(rr)))

600 lines = [", ".join("%s: %d" % (k, v) for k, v in sorted(phases.items()))]

601 lines += problems

602 section("Storage (%d volumes)" % sum(phases.values()), lines)

603

604

605 def backups():

606 d = kc(["get", "snapshots.kopiur.home-operations.com", "-A"])

607 if d is None:

608 section("Backups", ["kopiur snapshots not readable"])

609 return

610 recent_ok = recent_fail = 0

611 for it in d.get("items", []):

612 a = age_days(it.get("metadata", {}).get("creationTimestamp"))

613 if a is None or a > 7:

614 continue

615 ph = str((it.get("status", {}) or {}).get("phase", ""))

616 if ph == "Failed":

617 recent_fail += 1

618 elif ph == "Succeeded":

619 recent_ok += 1

620 tot = recent_ok + recent_fail

621 rate = (100.0 * recent_ok / tot) if tot else 0.0

622 lines = ["last 7d: %d ok, %d failed (%.0f%% success)" % (recent_ok, recent_fail, rate)]

623

624 p = kc(["get", "snapshotpolicies.kopiur.home-operations.com", "-A"])

625 if p is not None:

626 stale = []

627 for it in p.get("items", []):

628 last = (it.get("status", {}) or {}).get("lastSuccessfulSnapshot")

629 a = age_days(last)

630 if a is None or a > 1.0:

631 stale.append("<code>%s/%s</code> (%s)" % (

632 h(it["metadata"]["namespace"]), h(it["metadata"]["name"]),

633 "never" if a is None else "%.1fd ago" % a))

634 if stale:

635 lines.append("no snapshot in 24h: " + ", ".join(stale[:8]))

636 section("Backups", lines)

637

638

639 def certs():

640 d = kc(["get", "certificates.cert-manager.io", "-A"])

641 if d is None:

642 section("Certificates", ["cert-manager certificates not readable"])

643 return

644 problems = []

645 for it in d.get("items", []):

646 ns, name = it["metadata"]["namespace"], it["metadata"]["name"]

647 st = it.get("status", {}) or {}

648 ready, reason, _ = ready_cond(it)

649 if ready != "True":

650 problems.append("<code>%s/%s</code> not Ready — %s"

651 % (h(ns), h(name), h(reason or ready)))

652 continue

653 overdue = age_days(st.get("renewalTime"))

654 if overdue is not None and overdue > 0:

655 problems.append("<code>%s/%s</code> renewal overdue by %.1fd"

656 % (h(ns), h(name), overdue))

657 continue

658 left = age_days(st.get("notAfter"))

659 if left is not None and -left < 2:

660 problems.append("<code>%s/%s</code> expires in %.1fd and has not renewed"

661 % (h(ns), h(name), -left))

662 section("Certificates (%d)" % len(d.get("items", [])), problems,

663 "all Ready and renewing on schedule")

664

665

666 def capacity():

667 nodes = kc(["get", "nodes"])

668 pods = kc(["get", "pods", "-A", "--field-selector", "status.phase=Running"])

669 if nodes is None or pods is None:

670 section("Capacity", ["nodes/pods not readable"])

671 return

672

673 def cpu(v):

674 if not v:

675 return 0.0

676 v = str(v)

677 return float(v[:-1]) / 1000.0 if v.endswith("m") else float(v)

678

679 def mem(v):

680 if not v:

681 return 0.0

682 v = str(v)

683 for suf, mult in (("Ki", 1 / 1024.0), ("Mi", 1.0), ("Gi", 1024.0)):

684 if v.endswith(suf):

685 return float(v[:-2]) * mult

686 try:

687 return float(v) / (1024 * 1024)

688 except Exception:

689 return 0.0

690

691 req_c = req_m = 0.0

692 for p in pods.get("items", []):

693 for c in p["spec"].get("containers", []):

694 r = (c.get("resources", {}) or {}).get("requests", {}) or {}

695 req_c += cpu(r.get("cpu"))

696 req_m += mem(r.get("memory"))

697 alloc_c = alloc_m = 0.0

698 for n in nodes.get("items", []):

699 a = n["status"].get("allocatable", {})

700 alloc_c += cpu(a.get("cpu"))

701 alloc_m += mem(a.get("memory"))

702

703 lines = [

704 "CPU requests %.1f / %.1f cores (%.0f%%)" % (

705 req_c, alloc_c, 100 * req_c / alloc_c if alloc_c else 0),

706 "Memory requests %.1f / %.1f GiB (%.0f%%)" % (

707 req_m / 1024, alloc_m / 1024, 100 * req_m / alloc_m if alloc_m else 0),

708 "%d running pods across %d nodes" % (

709 len(pods.get("items", [])), len(nodes.get("items", []))),

710 ]

711 section("Capacity", lines)

712

713

714 def renovate():

715 env = dict(os.environ)

716 if not env.get("GH_TOKEN") and env.get("HOMELAB_GH_TOKEN"):

717 env["GH_TOKEN"] = env["HOMELAB_GH_TOKEN"]

718 try:

719 p = subprocess.run(

720 ["gh", "pr", "list", "--repo", "janpuc/home-ops",

721 "--state", "open", "--limit", "50",

722 "--json", "number,title,isDraft,createdAt"],

723 capture_output=True, text=True, timeout=60, env=env,

724 )

725 if p.returncode != 0:

726 section("Renovate / PRs", ["gh pr list failed"])

727 return

728 prs = json.loads(p.stdout or "[]")

729 except Exception:

730 section("Renovate / PRs", ["gh unavailable"])

731 return

732

733 if not prs:

734 section("Renovate / PRs", [], "no open PRs")

735 return

736 lines = ["%d open" % len(prs)]

737 for pr in sorted(prs, key=lambda x: x.get("createdAt") or "")[:10]:

738 a = age_days(pr.get("createdAt"))

739 lines.append("#%s %s%s" % (

740 pr["number"], h(pr["title"][:66]),

741 " (%.0fd old)" % a if a and a > 7 else ""))

742 section("Renovate / PRs", lines)

743

744

745 OUT.append("Homelab weekly digest — %s "

746 % NOW.strftime("%Y-%m-%d %H:%M UTC"))

747 for fn in (flux, storage, backups, certs, capacity, renovate):

748 try:

749 fn()

750 except Exception as e:

751 OUT.append("%s   section failed: %s "

752 % (h(fn.__name__), h(type(e).__name__)))

753

754 sys.stdout.write("".join(OUT))

755

756 weekly-digest-run.sh: |-

757 #!/bin/sh

758 set -eu

759 export PATH=/opt/hermes/.venv/bin:/opt/data/.local/bin:/opt/data/.local/homebrew/bin:/usr/local/bin:/usr/bin:/bin

760 weekly-digest | matrix-post digest --html >/dev/null

761

762 home-ops-guard.py: |-

763 """Block git write operations inside a home-ops checkout.

764

765 Wire protocol (agent/shell_hooks.py): a JSON payload arrives on stdin;

766 printing {"decision": "block", "reason": ...} refuses the call. Any other

767 output - or none - lets it through. Failing open is deliberate: a broken

768 guard must not wedge the agent.

769 """

770 import json

771 import shlex

772 import sys

773

774 WRITE_SUBCOMMANDS = {

775 "commit", "push", "merge", "rebase", "cherry-pick", "revert",

776 "reset", "clean", "am", "apply", "restore", "checkout", "switch",

777 "stash", "filter-branch", "update-ref", "gc", "prune", "mv", "rm",

778 }

779 CONDITIONAL = {"tag": ("-d", "-D", "--delete"), "branch": ("-D", "--delete")}

780

781 SPLITTERS = ("&&", "||", ";", "|", "\n")

782

783

784 def segments(command):

785 """Split a compound shell command into individual invocations."""

786 parts = [command]

787 for sep in SPLITTERS:

788 nxt = []

789 for chunk in parts:

790 nxt.extend(chunk.split(sep))

791 parts = nxt

792 return [p.strip() for p in parts if p.strip()]

793

794

795 def offending_subcommand(segment):

796 """Return the git subcommand if this segment is a git write, else None."""

797 try:

798 tokens = shlex.split(segment)

799 except ValueError:

800 return None

801 while tokens and ("=" in tokens[0].split(" ")[0] or tokens[0] == "sudo"):

802 tokens.pop(0)

803 if not tokens or tokens[0] != "git":

804 return None

805 rest = tokens[1:]

806 while rest and rest[0].startswith("-"):

807 if rest[0] in ("-C", "-c", "--git-dir", "--work-tree"):

808 rest = rest[2:]

809 else:

810 rest = rest[1:]

811 if not rest:

812 return None

813 sub = rest[0]

814 if sub in WRITE_SUBCOMMANDS:

815 return sub

816 flags = CONDITIONAL.get(sub)

817 if flags and any(f in rest[1:] for f in flags):

818 return sub

819 return None

820

821

822 def main():

823 try:

824 payload = json.load(sys.stdin)

825 except Exception:

826 return

827 if payload.get("tool_name") != "terminal":

828 return

829 command = (payload.get("tool_input") or {}).get("command") or ""

830 cwd = payload.get("cwd") or ""

831

832 if "home-ops" not in cwd and "home-ops" not in command:

833 return

834

835 for segment in segments(command):

836 sub = offending_subcommand(segment)

837 if sub:

838 print(json.dumps({

839 "decision": "block",

840 "reason": (

841 "Blocked 'git " + sub + "' in a home-ops checkout. "

842 "home-ops is read-only for the agent: pull first, never "

843 "auto-commit, and only edit when explicitly asked. Report "

844 "the change you want to make and let the user run it, or "

845 "ask for confirmation first."

846 ),

847 }))

848 return

849

850

851 main()

cron_mode is binary; the deny list is the actual safety

gotcha ·ok L162

cron_mode is binary: "deny" blocks every approval-gated command in a cron session, "approve" auto-approves ALL of them. approvals.mode: smart above does NOT apply to cron — approval.py::_get_cron_approval_mode only ever returns deny or approve. It is set to approve so scheduled jobs can actually act (remediate, commit, restart) instead of only reading.

What makes that safe is the deny list immediately below, not the approval mode. Read that annotation before changing either.

2026-08-0401KZ6MC7M0AMFZ093K3MEHC6AF

terminal deny list is the only thing protecting cron jobs

invariant ·ok L163

This list is the guardrail that makes cron_mode: approve safe, and it is the only thing standing between a scheduled job and an unrecoverable cluster. These fnmatch globs are matched against terminal commands BEFORE the approve / --yolo / mode:off bypass, so they hold even under cron_mode: approve. Matching is case-insensitive.

Show the rest

The groups, in order: cluster-destroying Talos operations (reset, wipe, upgrade); irreversible git history rewrites and working-tree destruction (force push, reset --hard, clean -f, filter-branch); Flux and Helm teardown — pruning is fine, deleting the controllers is not; blanket kubectl deletes including namespaces, PVCs and PVs; storage and filesystem destruction (mkfs, rm -rf /, dd to a device, wipefs); and piping the internet into a shell.

Entries should be removed only with a specific reason. Adding to a group is cheap; deleting from one silently widens what a cron session may do.

2026-08-0401KZ6MCC75ZHTH0MVN4SXGM89Y

pre_tool_call hook names the python interpreter explicitly

why ·ok L186

Shell hooks live in agent/shell_hooks.py. A pre_tool_call hook refuses a tool call by printing {"decision": "block", "reason": "..."} on stdout.

The interpreter is named explicitly because commands run through shlex.split with shell=False — a shebang plus exec bit would not be honoured. That is also why the init container chmods the script 644 rather than 755.

2026-08-0401KZ6MCEY6DFMJBVYSDN7BWPPF

hooks_auto_accept is required or headless hooks never register

gotcha ·ok L191

Required, not a convenience. Registration normally prompts for consent on first use of each (event, command) pair and records the answer in ~/.hermes/shell-hooks-allowlist.json. The gateway and cron have no TTY, so without this the hook silently never registers — the guard appears configured and simply never runs.

2026-08-0401KZ6MCJ3RTNR4RAA4T4DKKQZF

aggregator stays on a different provider from the four advisors

why ·ok L226

The council's four reference models are four distinct pretraining lineages — Moonshot, Zhipu, DeepSeek, Qwen — chosen so the members fail differently rather than for individual quality; all four are served by opencode-go.

The aggregator is deliberately a fifth family on a DIFFERENT provider. That keeps aggregation working through an opencode-go outage and stops it being biased toward any one reference. Consolidating it onto opencode-go would quietly remove both properties.

2026-08-0401KZ6MCMZT45K2WX8TS2JCK4EX

council preset fans out per user turn, not per tool iteration

why ·ok L231

fanout: user_turn runs the advisors ONCE per user turn, after which MiniMax-M3 works the task alone. The default, per_iteration, re-polls all four advisors after every tool result — a 25-iteration kubectl/terminal task would cost 100 advisor calls instead of 4.

reference_max_tokens: 600 is there because advisor generation dominates MoA turn latency (roughly 0.88 correlation with output tokens) and the aggregator only needs the gist of each judgement, not a full answer.

Show the rest

max_tokens is kept for forward compatibility only: as of v2026.7.7 the preset's max_tokens does NOT cap the aggregator — moa_loop passes the caller's value straight through, so the acting model uses its real maximum. Do not rely on this number to bound cost.

The `quick` preset below is the same idea at three calls instead of five, for when a council is overkill.

2026-08-0401KZ6MCR8FGV36E8C97PJEV386

quick preset can afford per_iteration fanout with two advisors

why ·ok L246

`quick` is deliberately left on the default per_iteration fanout, unlike `council` above. With only two advisors, refreshing their advice on every iteration stays cheap — the cost blow-up that forced council onto user_turn does not apply here.

2026-08-0401KZ6MDHMJRC6VVZY7AE88TZ9E

split MCP entries per server to stop registering all 418 tools

why ·ok L259

One Hermes entry per upstream MCP server, rather than a single aggregated "homelab" entry pointing at http://litellm.ai:4000/mcp. Two reasons:

Show the rest

1. The aggregate registered all 418 tools unfiltered — mcp_tool.py _register_server_tools treats "neither include nor exclude set" as register-everything — costing roughly 154k tokens of schema JSON on EVERY request. Note that `hermes prompt-size` does NOT reveal this: it runs offline and only counts the 35 built-in tools, so the cost was invisible from the obvious diagnostic. 2. Split entries become separate `mcp-<name>` toolsets, which can later be scoped per platform via platform_toolsets.

The URL form matters: LiteLLM routes a single server at /mcp/<name>/ and the TRAILING SLASH is required — /mcp/<name>/mcp returns 0 tools with no error.

Splitting also renamed every tool: mcp__homelab__kubectl_get_pods became mcp__kubectl__kubectl_get_pods. Skills under /opt/data/skills were rewritten to match, so reverting this shape breaks them.

2026-08-0401KZ6MDPVRCMZT612X1EKN2E31

kubectl MCP include list trims families this cluster does not run

why ·ok L264

This include list is a token-budget decision, not a capability one: 275 tools upstream (~44.9k tokens) down to 95 (~16.8k). What was dropped is whole families this cluster does not run — kind, vind, kubevirt, istio, capi, keda, argo-rollouts, flagger, velero backup/restore, multi-cluster — plus the roughly 35 helm_* tools, since Flux owns Helm here.

Adding a tool back is fine; adding a family back costs real tokens on every request.

2026-08-0401KZ6MDT18PH5S1YRXZYTF8P7Q

HA MCP include list keeps only read and the skill-needed tools

why ·ok L361

Home Assistant was the biggest offender by bytes: 78 tools at roughly 72.2k tokens, driven by the config_set_* and manage_* schemas — ha_manage_addon alone is 15.8 KB. The include list below keeps the read/query surface plus the service-call and todo tools that the ha-car-status and ev-charge-reminder skills need, bringing it to about 14.4k tokens.

Be specific when adding here: one manage_* tool can cost more than the entire kept set.

2026-08-0401KZ6MEGK2PSA8H51S6SMPSHDH

matrix-post.py delivers into named Matrix rooms from anywhere

why ·ok L412

Delivers a message into a named Matrix room from anywhere the agent runs — cron jobs, skills, scripts. The init container installs it as /opt/data/.local/bin/matrix-post, which is already on PATH.

It replaces the file-delivery-to-chat approach of shelling out to kubectl to scrape MATRIX_ACCESS_TOKEN from the running hermes-matrix pod: the credentials now arrive as env from the hermes-matrix-access Secret, so nothing is coupled to a pod name.

Show the rest

Rooms are addressed by alias (cluster, notify, alerts, digest, ...) which resolves to MATRIX_ROOM_<ALIAS>. That is what keeps the private room IDs out of every caller and out of git.

This script deliberately contains no "${...}" sequences. Flux postBuild substitution runs over this ConfigMap and would eat them.

2026-08-0401KZ6MF06444HY8QPH5PF5GACR

body stays plain text so non-HTML Matrix clients render cleanly

why ·ok L478

The plain `body` field is left as-is and doubles as the plain-text fallback for clients that do not render HTML — only formatted_body carries the markup. Overwriting body with the HTML would make those clients show tags.

2026-08-0401KZ6MFQGM41PGMG9P23TSX3FC

matrix-post error path never echoes the bearer token

invariant ·ok L492

Never surface the access token in an error path. The HTTP response body is safe to show and is genuinely useful for diagnosis, so it is included; the generic handler below deliberately reports only the exception type, because an arbitrary exception's str() can carry the request headers and therefore the bearer token.

2026-08-0401KZ6MFTH9QB7EGXX060EMKEVA

weekly-digest.py is deterministic on purpose, not an agent turn

why ·ok L499

Weekly homelab rollup, emitted as HTML on stdout and delivered by the weekly-digest-run wrapper. It is deterministic on purpose: a factual rollup is cheaper and more reliable as a script than as an agent turn, and the #digest room persona is there to handle follow-up questions about it.

Every section degrades to a one-line note rather than raising, so a missing RBAC grant or an uninstalled CRD costs one section instead of the whole digest.

Like matrix-post.py, this deliberately contains no "${...}" sequences — Flux postBuild substitution runs over this ConfigMap.

2026-08-0401KZ6MFXNSCF3Q4698KXX59N4Q

snapshot freshness must read lastSuccessfulSnapshot, not time

gotcha ·ok L628

The field is lastSuccessfulSnapshot, not lastSnapshotTime — the latter exists and updates on failed attempts, so reading it would report healthy backups that never actually succeeded.

The 1.0-day threshold is derived from cadence: snapshots run every 2h, so 24h without a successful one is genuinely stale rather than merely unlucky.

2026-08-0401KZ6MGKVXZFR0V2H54KJRX53Z

cert health judged by cert-manager renewal, not expiry window

why ·ok L653

Certificate health is judged against cert-manager's own renewalTime and Ready condition rather than a fixed "expires within 30 days" window. These are 7-day certificates, so a 30-day window would fire every single week on a perfectly healthy cert. The question worth asking is whether cert-manager is keeping up, not how close expiry is.

2026-08-0401KZ6MGRJQG2ZB3V830S0ZR6JY

GH_TOKEN mapped from HOMELAB_GH_TOKEN so a rebuilt PVC still works

why ·ok L716

gh reads GH_TOKEN or GITHUB_TOKEN, but the pod only exposes HOMELAB_GH_TOKEN (named that way to keep it out of gh's implicit reach elsewhere). Mapping it across here is deliberate — the alternative is depending on `gh auth login` state persisted on the PVC, which a rebuilt volume would silently lose.

2026-08-0401KZ6MJ10RTWJHBP3306PRVV7K

weekly-digest-run stays silent so

why ·ok L756

The cron entrypoint. It prints nothing on success — stdout is redirected to /dev/null — so the job stays silent unless delivery actually fails. That is the point: the #notify room is for output, #digest receives the rollup itself, and a healthy run should be invisible. Adding logging here would post noise into a room chosen for being unmuted.

2026-08-0401KZ6MJ43JWBMQSSZ9TJB6WB4Q

home-ops-guard.py enforces pull-first as a hard gate

why ·ok L762

Enforces the standing home-ops convention — pull first, never auto-commit, edit only when asked — as a hard gate rather than relying on the agent remembering it. The init container copies it to /opt/data/hook-scripts/ and config.yaml registers it as the pre_tool_call hook.

Like the other embedded scripts here it deliberately contains no "${...}" sequences, because Flux postBuild substitution runs over this ConfigMap.

2026-08-0401KZ6MJ713J2BDV5XHK95QR7FK

WRITE_SUBCOMMANDS is allow-by-omission; new git verbs slip through

invariant ·ok L774

WRITE_SUBCOMMANDS is every git subcommand that mutates history, refs or the working tree — unconditionally blocked. CONDITIONAL below holds the two that are destructive only with a delete or force flag (tag, branch) and read-only otherwise, so they are matched on their flags instead.

Both sets are allow-by-omission: a subcommand missing from here passes the guard. That is the correct failure direction for a hook that must not wedge the agent, but it means adding a new mutating git subcommand to the list is the only way it gets blocked.

2026-08-0401KZ6MJB6TC6JXEZYYXWQYB01W

guard parses env prefixes and git's own -C/-c globals first

gotcha ·ok L801

Both of these loops exist to stop trivial evasion of the guard. The first strips env-var prefixes and a sudo wrapper, so `FOO=bar git push` and `sudo git push` still resolve to git. The second skips git's own global flags — importantly -C, -c, --git-dir and --work-tree consume a following argument, so they are advanced by two rather than one; without that, `git -C /repo push` would present "-C" as the subcommand and slip through.

2026-08-0401KZ6MJDXP2J2950VKJM8CSVP7

home-ops-guard scopes the block to home-ops only

why ·ok L832

Scoped to home-ops only — other repos in the workspace are fair game and the agent is expected to commit freely in them. The check is deliberately loose (cwd OR the command text) so a `git -C .../home-ops push` issued from elsewhere is still caught.

2026-08-0401KZ6MJGB2BDYSN0R13MQ1G0HQ

koment MCP reaches hermes directly without a LiteLLM hop

why ·ok L398

The one MCP server here that does not go through LiteLLM, because koment is self-hosted in this cluster and LiteLLM would add a proxy hop with nothing to contribute. KOMENT_API_KEY is a scoped bearer whose SHA-256 hash lives in koment's credentials.yaml, not a LiteLLM key.

Show the rest

Over this transport hermes gets koment_get, koment_search, koment_repositories and — because the credential carries write scope — koment_add, which opens a review pull request rather than committing. koment_reanchor, koment_convert_comment and koment_acknowledge_comment are stdio-only and deliberately out of reach, which suits this repo: convert has no YAML detector and acknowledge is forbidden by AGENTS.md.

2026-08-0401KZ6PHQ7KD2EAE6CEN7D9PK50

Aux models: hy3 for volume, flash-vision-exp for judgment — benchmarked

why ·ok L117

The auxiliary assignments follow a 2026-08-22 benchmark of all 23 models (latency, exact-match reasoning, strict JSON, tool-calls, format compliance, 17k-token needle), run after deepseek-v4-flash — the previous model for all of these — went 403 RegionError on opencode.

Show the rest

The split is deliberate: high-volume low-stakes tasks (session_search, title_generation, profile_describer, triage_specifier) run on hy3, which passed every compliance task at ~1/10th the cost of anything comparable; judgment-heavy tasks (goal_judge, kanban_decomposer, curator) and delegation run on deepseek-v4-flash-vision-exp, which scored 9/9 including verified tool-calling. approval, compression, web_extract and vision stay on minimax/MiniMax-M3, which won the benchmark outright (TTFT 1.3s, 113 tok/s, needle in 1.7s) — approval in particular has a 60s timeout that M3 clears with a 1.7s median.

Do not put hy3 on a vision path: it is text-only but answers image questions confidently instead of erroring.

2026-08-2201M0M62NCR46Q1A6QVGMN0PZ9M

kimi-k2.6, not k3: the council seat is latency-bound, same lineage

why ·ok L219

The Moonshot council seat runs kimi-k2.6 rather than the newer kimi-k3 on purpose. In the 2026-08-22 benchmark k3 took ~29s to first token even on a trivial prompt (it always reasons deeply), threw a transient 500, and missed the tool-call task; k2.6 answered everything cleanly at 2.1s TTFT. Council advisors are capped at 600 tokens and fan out once per user turn, so the seat is latency-bound, not depth-bound — the aggregator only needs the gist. Swapping k3 back in makes every council turn wait half a minute for its slowest member. k3 stays available for explicit use via the `k3` alias.

Show the rest

The `glm` alias two blocks up pins glm-5.2 for the same reason: 5.3 benchmarked at 30 tok/s vs 74 for 5.2 (needle 13.8s vs 4.5s) with no measured accuracy gain, so the interactive alias keeps the fast one and 5.3 is reachable by full name.

2026-08-2201M0M62XPZQ2R2KM1AAH2N6XSW