feat(ops): web + worker container healthchecks (#20)
docker-compose only health-checked db. Add: - web: an unauthenticated GET /api/health (excluded from the auth middleware) + a compose healthcheck that hits it (node global fetch — no curl/wget in image). - worker: a background daemon thread touches /tmp/worker.alive every ~15s (independent of the main loop, so it stays fresh through a long download) + a compose healthcheck on the file's mtime (unhealthy if >90s stale). Now `docker compose ps` reports real health for all services. worker 223 tests, web 154, tsc + build + compose config clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -24,9 +24,19 @@ services:
|
|||||||
LYRA_SECRET_KEY: ${LYRA_SECRET_KEY}
|
LYRA_SECRET_KEY: ${LYRA_SECRET_KEY}
|
||||||
# Optional shared-password gate for the whole UI/API. Unset ⇒ app is open. See README.
|
# Optional shared-password gate for the whole UI/API. Unset ⇒ app is open. See README.
|
||||||
LYRA_PASSWORD: ${LYRA_PASSWORD:-}
|
LYRA_PASSWORD: ${LYRA_PASSWORD:-}
|
||||||
|
# Next's standalone server binds to HOSTNAME; default is the container hostname (its own
|
||||||
|
# IP only), so loopback healthchecks can't reach it. Bind all interfaces instead.
|
||||||
|
HOSTNAME: 0.0.0.0
|
||||||
ports:
|
ports:
|
||||||
# host:container — host 8770 avoids the Quill app on 3000 (and Lidarr on 8686)
|
# host:container — host 8770 avoids the Quill app on 3000 (and Lidarr on 8686)
|
||||||
- "8770:3000"
|
- "8770:3000"
|
||||||
|
healthcheck:
|
||||||
|
# unauthenticated liveness endpoint; node 22 has global fetch (no curl/wget in the image)
|
||||||
|
test: ["CMD-SHELL", "node -e \"fetch('http://127.0.0.1:3000/api/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))\""]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 20s
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
@@ -55,6 +65,14 @@ services:
|
|||||||
# empty local dir so the mount is always valid; Soulseek delivery only works once
|
# empty local dir so the mount is always valid; Soulseek delivery only works once
|
||||||
# this points at the real slskd downloads directory.
|
# this points at the real slskd downloads directory.
|
||||||
- ${SLSKD_DOWNLOADS_DIR:-./slskd-downloads}:/slskd-downloads
|
- ${SLSKD_DOWNLOADS_DIR:-./slskd-downloads}:/slskd-downloads
|
||||||
|
healthcheck:
|
||||||
|
# the worker touches /tmp/worker.alive every ~15s from a background thread (survives long
|
||||||
|
# downloads); unhealthy if it hasn't been touched in 90s (process wedged/exited).
|
||||||
|
test: ["CMD-SHELL", "test $(( $(date +%s) - $(stat -c %Y /tmp/worker.alive 2>/dev/null || echo 0) )) -lt 90"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
start_period: 20s
|
||||||
depends_on:
|
depends_on:
|
||||||
db:
|
db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import { describe, it, expect } from "vitest";
|
||||||
|
import { GET } from "./route";
|
||||||
|
|
||||||
|
describe("GET /api/health", () => {
|
||||||
|
it("returns 200 ok", async () => {
|
||||||
|
const res = GET();
|
||||||
|
expect(res.status).toBe(200);
|
||||||
|
expect(await res.json()).toEqual({ ok: true });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
// Unauthenticated liveness endpoint for the docker-compose healthcheck. Excluded from the
|
||||||
|
// auth gate in middleware.ts.
|
||||||
|
export function GET() {
|
||||||
|
return Response.json({ ok: true });
|
||||||
|
}
|
||||||
@@ -8,7 +8,10 @@ export async function middleware(req: NextRequest) {
|
|||||||
if (!token) return NextResponse.next(); // auth disabled (no LYRA_PASSWORD)
|
if (!token) return NextResponse.next(); // auth disabled (no LYRA_PASSWORD)
|
||||||
|
|
||||||
const { pathname } = req.nextUrl;
|
const { pathname } = req.nextUrl;
|
||||||
if (pathname === "/login" || pathname === "/api/auth") return NextResponse.next();
|
// /api/health is the unauthenticated liveness probe for the compose healthcheck.
|
||||||
|
if (pathname === "/login" || pathname === "/api/auth" || pathname === "/api/health") {
|
||||||
|
return NextResponse.next();
|
||||||
|
}
|
||||||
|
|
||||||
const cookie = req.cookies.get(AUTH_COOKIE)?.value ?? "";
|
const cookie = req.cookies.get(AUTH_COOKIE)?.value ?? "";
|
||||||
if (safeEqual(cookie, token)) return NextResponse.next();
|
if (safeEqual(cookie, token)) return NextResponse.next();
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
import threading
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
|
|
||||||
@@ -203,8 +204,25 @@ def _require_secret_key() -> None:
|
|||||||
print(f"\n{banner}\nWARNING: {msg}.\n{banner}\n", flush=True)
|
print(f"\n{banner}\nWARNING: {msg}.\n{banner}\n", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
HEARTBEAT_FILE = "/tmp/worker.alive"
|
||||||
|
|
||||||
|
|
||||||
|
def _liveness_heartbeat() -> None:
|
||||||
|
"""Touch HEARTBEAT_FILE every HEARTBEAT_SECONDS from a background daemon thread, so the
|
||||||
|
compose healthcheck sees the process as alive even while the main loop is blocked in a
|
||||||
|
long download (a loop-driven heartbeat would go stale and falsely report unhealthy)."""
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
with open(HEARTBEAT_FILE, "w") as fh:
|
||||||
|
fh.write("1")
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
time.sleep(HEARTBEAT_SECONDS)
|
||||||
|
|
||||||
|
|
||||||
def run_forever() -> None:
|
def run_forever() -> None:
|
||||||
_require_secret_key()
|
_require_secret_key()
|
||||||
|
threading.Thread(target=_liveness_heartbeat, daemon=True).start()
|
||||||
conn = wait_for_db()
|
conn = wait_for_db()
|
||||||
clear_staging_root(STAGING_ROOT) # sweep any staging dirs orphaned by a prior crash
|
clear_staging_root(STAGING_ROOT) # sweep any staging dirs orphaned by a prior crash
|
||||||
reclaimed = reclaim_stuck_jobs(conn) # requeue jobs a prior crash left mid-pipeline
|
reclaimed = reclaim_stuck_jobs(conn) # requeue jobs a prior crash left mid-pipeline
|
||||||
|
|||||||
Reference in New Issue
Block a user