feat: monitor sweep, real MB browser, and worker-loop wiring
This commit is contained in:
@@ -3,10 +3,12 @@ import time
|
||||
from lyra_worker.claim import claim_next
|
||||
from lyra_worker.config import get_config
|
||||
from lyra_worker.db import wait_for_db
|
||||
from lyra_worker.monitor import MonitorConfig, reconcile, sweep
|
||||
from lyra_worker.pipeline import run_pipeline
|
||||
from lyra_worker.registry import build_adapters, build_resolver, build_tagger
|
||||
from lyra_worker.registry import build_adapters, build_browser, build_resolver, build_tagger
|
||||
|
||||
IDLE_SLEEP = 2.0
|
||||
MONITOR_TICK_SECONDS = 60.0
|
||||
|
||||
|
||||
def run_forever() -> None:
|
||||
@@ -14,16 +16,34 @@ def run_forever() -> None:
|
||||
adapters = build_adapters(get_config(conn))
|
||||
resolver = build_resolver()
|
||||
tagger = build_tagger()
|
||||
browser = build_browser()
|
||||
print(f"worker: {len(adapters)} adapter(s) enabled: {[a.name for a in adapters]}", flush=True)
|
||||
print("worker: waiting for jobs", flush=True)
|
||||
last_tick = 0.0
|
||||
try:
|
||||
while True:
|
||||
mcfg = MonitorConfig.from_config(get_config(conn))
|
||||
now = time.monotonic()
|
||||
if mcfg.enabled and now - last_tick >= MONITOR_TICK_SECONDS:
|
||||
try:
|
||||
sweep(conn, browser, mcfg)
|
||||
except Exception as e: # a monitor error must never kill the worker
|
||||
print(f"worker: monitor sweep failed: {e}", flush=True)
|
||||
conn.rollback()
|
||||
last_tick = now
|
||||
|
||||
job_id = claim_next(conn)
|
||||
if job_id is None:
|
||||
time.sleep(IDLE_SLEEP)
|
||||
continue
|
||||
print(f"worker: claimed job {job_id}", flush=True)
|
||||
run_pipeline(conn, job_id, adapters, resolver=resolver, tagger=tagger)
|
||||
if mcfg.enabled:
|
||||
try:
|
||||
reconcile(conn, job_id, mcfg)
|
||||
except Exception as e:
|
||||
print(f"worker: reconcile failed for job {job_id}: {e}", flush=True)
|
||||
conn.rollback()
|
||||
print(f"worker: finished job {job_id}", flush=True)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
Reference in New Issue
Block a user