From eb51489ba93ed6471b45533970d98677c1a5ff0b Mon Sep 17 00:00:00 2001 From: Jonathan Date: Tue, 14 Jul 2026 13:28:26 +0200 Subject: [PATCH] feat(ops): web + worker container healthchecks (#20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docker-compose only health-checked db. Add: - web: an unauthenticated GET /api/health (excluded from the auth middleware) + a compose healthcheck that hits it (node global fetch — no curl/wget in image). - worker: a background daemon thread touches /tmp/worker.alive every ~15s (independent of the main loop, so it stays fresh through a long download) + a compose healthcheck on the file's mtime (unhealthy if >90s stale). Now `docker compose ps` reports real health for all services. worker 223 tests, web 154, tsc + build + compose config clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker-compose.yml | 18 ++++++++++++++++++ web/src/app/api/health/route.test.ts | 10 ++++++++++ web/src/app/api/health/route.ts | 5 +++++ web/src/middleware.ts | 5 ++++- worker/lyra_worker/main.py | 18 ++++++++++++++++++ 5 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 web/src/app/api/health/route.test.ts create mode 100644 web/src/app/api/health/route.ts diff --git a/docker-compose.yml b/docker-compose.yml index 8d415aa..9b65646 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,9 +24,19 @@ services: LYRA_SECRET_KEY: ${LYRA_SECRET_KEY} # Optional shared-password gate for the whole UI/API. Unset ⇒ app is open. See README. LYRA_PASSWORD: ${LYRA_PASSWORD:-} + # Next's standalone server binds to HOSTNAME; default is the container hostname (its own + # IP only), so loopback healthchecks can't reach it. Bind all interfaces instead. + HOSTNAME: 0.0.0.0 ports: # host:container — host 8770 avoids the Quill app on 3000 (and Lidarr on 8686) - "8770:3000" + healthcheck: + # unauthenticated liveness endpoint; node 22 has global fetch (no curl/wget in the image) + test: ["CMD-SHELL", "node -e \"fetch('http://127.0.0.1:3000/api/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))\""] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s depends_on: db: condition: service_healthy @@ -55,6 +65,14 @@ services: # empty local dir so the mount is always valid; Soulseek delivery only works once # this points at the real slskd downloads directory. - ${SLSKD_DOWNLOADS_DIR:-./slskd-downloads}:/slskd-downloads + healthcheck: + # the worker touches /tmp/worker.alive every ~15s from a background thread (survives long + # downloads); unhealthy if it hasn't been touched in 90s (process wedged/exited). + test: ["CMD-SHELL", "test $(( $(date +%s) - $(stat -c %Y /tmp/worker.alive 2>/dev/null || echo 0) )) -lt 90"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s depends_on: db: condition: service_healthy diff --git a/web/src/app/api/health/route.test.ts b/web/src/app/api/health/route.test.ts new file mode 100644 index 0000000..157b9bd --- /dev/null +++ b/web/src/app/api/health/route.test.ts @@ -0,0 +1,10 @@ +import { describe, it, expect } from "vitest"; +import { GET } from "./route"; + +describe("GET /api/health", () => { + it("returns 200 ok", async () => { + const res = GET(); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ok: true }); + }); +}); diff --git a/web/src/app/api/health/route.ts b/web/src/app/api/health/route.ts new file mode 100644 index 0000000..ca5b32e --- /dev/null +++ b/web/src/app/api/health/route.ts @@ -0,0 +1,5 @@ +// Unauthenticated liveness endpoint for the docker-compose healthcheck. Excluded from the +// auth gate in middleware.ts. +export function GET() { + return Response.json({ ok: true }); +} diff --git a/web/src/middleware.ts b/web/src/middleware.ts index 742251f..fa40c22 100644 --- a/web/src/middleware.ts +++ b/web/src/middleware.ts @@ -8,7 +8,10 @@ export async function middleware(req: NextRequest) { if (!token) return NextResponse.next(); // auth disabled (no LYRA_PASSWORD) const { pathname } = req.nextUrl; - if (pathname === "/login" || pathname === "/api/auth") return NextResponse.next(); + // /api/health is the unauthenticated liveness probe for the compose healthcheck. + if (pathname === "/login" || pathname === "/api/auth" || pathname === "/api/health") { + return NextResponse.next(); + } const cookie = req.cookies.get(AUTH_COOKIE)?.value ?? ""; if (safeEqual(cookie, token)) return NextResponse.next(); diff --git a/worker/lyra_worker/main.py b/worker/lyra_worker/main.py index 59a07e9..0e5c1d8 100644 --- a/worker/lyra_worker/main.py +++ b/worker/lyra_worker/main.py @@ -1,5 +1,6 @@ import os import sys +import threading import time import uuid @@ -203,8 +204,25 @@ def _require_secret_key() -> None: print(f"\n{banner}\nWARNING: {msg}.\n{banner}\n", flush=True) +HEARTBEAT_FILE = "/tmp/worker.alive" + + +def _liveness_heartbeat() -> None: + """Touch HEARTBEAT_FILE every HEARTBEAT_SECONDS from a background daemon thread, so the + compose healthcheck sees the process as alive even while the main loop is blocked in a + long download (a loop-driven heartbeat would go stale and falsely report unhealthy).""" + while True: + try: + with open(HEARTBEAT_FILE, "w") as fh: + fh.write("1") + except OSError: + pass + time.sleep(HEARTBEAT_SECONDS) + + def run_forever() -> None: _require_secret_key() + threading.Thread(target=_liveness_heartbeat, daemon=True).start() conn = wait_for_db() clear_staging_root(STAGING_ROOT) # sweep any staging dirs orphaned by a prior crash reclaimed = reclaim_stuck_jobs(conn) # requeue jobs a prior crash left mid-pipeline