feat(worker): chunk the library scan + reconnect on DB loss
Two worker-loop robustness fixes: - Scan chunking: maybe_run_scan advances the library scan by at most one scan_chunk (default scan.chunkSize=25 albums) per loop iteration, so a large library no longer blocks job-claim/monitor for minutes-to-hours. Progress persists in scan.inProgress/scan.cursor/scan.progress and is resumable; scan.result (unchanged format) is written on completion. - DB reconnect: wrap the loop body in `except psycopg.OperationalError` (AdminShutdown subclasses it) to close the dead handle and reconnect via wait_for_db() in-process, instead of crashing and relying on the container restart policy. Verified via a backend-termination test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
import psycopg
|
||||
import pytest
|
||||
|
||||
from lyra_worker.db import connect, wait_for_db
|
||||
|
||||
|
||||
def test_terminated_backend_raises_operational_error_then_reconnects(conn):
|
||||
"""The run_forever reconnect boundary catches psycopg.OperationalError and re-runs
|
||||
wait_for_db(). This exercises both primitives: a server-side backend termination (the
|
||||
real AdminShutdown case) surfaces as OperationalError, and wait_for_db reconnects."""
|
||||
victim = connect()
|
||||
try:
|
||||
with victim.cursor() as cur:
|
||||
cur.execute("SELECT pg_backend_pid()")
|
||||
pid = cur.fetchone()[0]
|
||||
with conn.cursor() as cur: # terminate the victim's backend, as a DB restart would
|
||||
cur.execute("SELECT pg_terminate_backend(%s)", (pid,))
|
||||
conn.commit()
|
||||
with pytest.raises(psycopg.OperationalError): # the exact type the loop catches
|
||||
with victim.cursor() as cur:
|
||||
cur.execute("SELECT 1")
|
||||
cur.fetchone()
|
||||
finally:
|
||||
try:
|
||||
victim.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
fresh = wait_for_db() # reconnect exactly as run_forever does
|
||||
try:
|
||||
with fresh.cursor() as cur:
|
||||
cur.execute("SELECT 1")
|
||||
assert cur.fetchone()[0] == 1
|
||||
finally:
|
||||
fresh.close()
|
||||
Reference in New Issue
Block a user