Jonathan/Lyra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(worker): chunk the discovery sweep (one chunk per loop iteration)

Browse Source 
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jonathan
2026-07-13 00:57:00 +02:00
parent 5351cd029d
commit a67c552f29
2 changed files with 72 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
worker/lyra_worker/main.py
+30 -14
View File
@@ -6,7 +6,7 @@ import psycopg
from lyra_worker.claim import claim_next
from lyra_worker.config import get_config
from lyra_worker.db import wait_for_db
from lyra_worker.discovery import DiscoveryConfig, run_discovery
from lyra_worker.discovery import DiscoveryConfig, DiscoveryResult, run_discovery
from lyra_worker.library import clear_staging_root
from lyra_worker.monitor import MonitorConfig, fulfill_owned_releases, reconcile, sweep
from lyra_worker.pipeline import run_pipeline
@@ -96,12 +96,9 @@ def maybe_run_scan(conn, resolver, probe, browser, config, dest_root: str = DEST
_set_config(conn, "scan.progress", f"{imported_total}/{skipped_total}")
def _run_discovery(conn, sources, browser, config=None) -> None:
def _run_discovery(conn, sources, browser, config=None) -> DiscoveryResult:
cfg = DiscoveryConfig.from_config(config if config is not None else get_config(conn))
result = run_discovery(conn, sources, browser, cfg)
_set_config(conn, "discover.result", f"artists {result.artists}, albums {result.albums}")
_set_config(conn, "discover.requested", "false")
print(f"worker: discovery done — {result.artists} artists, {result.albums} albums", flush=True)
return run_discovery(conn, sources, browser, cfg)
def finish_job(conn, job_id, mcfg) -> None:
@@ -117,21 +114,40 @@ def finish_job(conn, job_id, mcfg) -> None:
def maybe_run_discovery(conn, sources, browser, config, dcfg, now, last_discover_tick,
tick_seconds: float = DISCOVER_TICK_SECONDS) -> float:
"""Run discovery at most once per loop iteration, whether triggered by the schedule
(enabled + interval elapsed) or a one-shot ``discover.requested`` flag. A single run
satisfies both, so the two triggers must not fire it twice off the same stale config
snapshot. Returns the (possibly advanced) last_discover_tick."""
"""Advance the discovery sweep by at most one chunk (``discover.chunkSize`` seeds) per call,
so the worker loop keeps claiming jobs between chunks. A sweep starts on ``discover.requested``
or a due schedule tick, then continues via ``discover.inProgress`` until a chunk drains 0 seeds
(every eligible seed refreshed), at which point ``discover.result`` is written. Returns the
(possibly advanced) last_discover_tick."""
requested = str(config.get("discover.requested", "")).strip().lower() in _TRUE
in_progress = str(config.get("discover.inProgress", "")).strip().lower() in _TRUE
due = dcfg.enabled and now - last_discover_tick >= tick_seconds
if not (requested or due):
if not (requested or due or in_progress):
return last_discover_tick
starting = not in_progress
if starting: # begin a fresh sweep
_set_config(conn, "discover.inProgress", "true")
_set_config(conn, "discover.requested", "false")
try:
_run_discovery(conn, sources, browser, config) # also clears discover.requested
result = _run_discovery(conn, sources, browser, config) # one chunk
except Exception as e: # a discovery error must never kill the worker
print(f"worker: discovery run failed: {e}", flush=True)
conn.rollback()
if requested:
_set_config(conn, "discover.requested", "false")
_set_config(conn, "discover.inProgress", "false") # abandon; a new request restarts it
_set_config(conn, "discover.requested", "false")
return now if due else last_discover_tick
a_prev, b_prev = (0, 0) if starting else _parse_progress(config.get("discover.progress", ""))
a_total, b_total = a_prev + result.artists, b_prev + result.albums
if result.seeds == 0: # every eligible seed refreshed -> sweep drained
_set_config(conn, "discover.result", f"artists {a_total}, albums {b_total}")
_set_config(conn, "discover.progress", "")
_set_config(conn, "discover.inProgress", "false")
print(f"worker: discovery done — {a_total} artists, {b_total} albums", flush=True)
else:
_set_config(conn, "discover.progress", f"{a_total}/{b_total}")
return now if due else last_discover_tick