Jonathan/Lyra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(worker): persisted scan worklist (O(N) scan, robust to mid-scan changes)

Browse Source 
Replace scan_chunk's O(N^2) tree re-walk-and-skip-to-cursor with a
persisted ScanWorkItem worklist: build_worklist walks the tree once at
scan start and inserts one row per album (idempotent via ON CONFLICT),
scan_chunk pops up to `limit` not-done rows by indexed cursor and marks
them done, and clear_worklist drops the scan's rows once drained.
scan_library is reimplemented on top of the worklist (one-shot scan_id,
drain, clear) with an unchanged signature/behavior. main.py's
maybe_run_scan now drives a generated scan.id through Config
(scan.id/scan.inProgress/scan.progress/scan.result) instead of a
scan.cursor string, so a mid-scan library change no longer risks
skipping or re-walking albums.
This commit is contained in:
Jonathan
2026-07-13 15:26:44 +02:00
parent de052a4ef7
commit 0115b82867
3 changed files with 136 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
worker/tests/test_scan_chunk.py
+47 -40
View File
@@ -1,13 +1,12 @@
from lyra_worker.adapters.fakes import FakeMbBrowser
from lyra_worker.browser import ReleaseGroupInfo
from lyra_worker.scan import scan_chunk
from lyra_worker.scan import build_worklist, scan_chunk, clear_worklist
from lyra_worker.types import MBTarget
from tests.test_scan import FakeProbe, FakeResolver, _album, _counts
def _tree(tmp_path):
"""Three albums across two artists, in nested-sorted order:
Artist A/Alpha, Artist A/Beta, Artist B/Gamma."""
"""Three albums across two artists: Artist A/Alpha, Artist A/Beta, Artist B/Gamma."""
_album(tmp_path, "Artist A", "Alpha (2001)")
_album(tmp_path, "Artist A", "Beta (2002)")
_album(tmp_path, "Artist B", "Gamma (2003)")
@@ -24,58 +23,66 @@ def _tree(tmp_path):
return resolver, browser
def test_scan_chunk_respects_limit_and_returns_cursor(conn, tmp_path):
resolver, browser = _tree(tmp_path)
cursor, imported, skipped, done = scan_chunk(
conn, resolver, FakeProbe(quality_class=2), browser, str(tmp_path), cursor="", limit=2)
assert (imported, skipped, done) == (2, 0, False) # processed 2 of 3, more remain
assert cursor == "Artist A/Beta (2002)" # last processed key
def test_build_worklist_inserts_one_row_per_album(conn, tmp_path):
resolver, _ = _tree(tmp_path)
n = build_worklist(conn, "scan1", str(tmp_path))
assert n == 3
with conn.cursor() as cur:
cur.execute('SELECT count(*) FROM "ScanWorkItem" WHERE "scanId" = %s AND NOT done', ("scan1",))
assert cur.fetchone()[0] == 3
def test_build_worklist_is_idempotent(conn, tmp_path):
resolver, _ = _tree(tmp_path)
assert build_worklist(conn, "scan1", str(tmp_path)) == 3
assert build_worklist(conn, "scan1", str(tmp_path)) == 0 # ON CONFLICT DO NOTHING
with conn.cursor() as cur:
cur.execute('SELECT count(*) FROM "ScanWorkItem" WHERE "scanId" = %s', ("scan1",))
assert cur.fetchone()[0] == 3
def test_scan_chunk_respects_limit_and_marks_done(conn, tmp_path):
resolver, browser = _tree(tmp_path)
build_worklist(conn, "scan1", str(tmp_path))
imported, skipped, done = scan_chunk(conn, resolver, FakeProbe(quality_class=2), browser, "scan1", limit=2)
assert (imported, skipped, done) == (2, 0, False) # 2 of 3 processed, more remain
with conn.cursor() as cur:
cur.execute('SELECT count(*) FROM "ScanWorkItem" WHERE "scanId" = %s AND done', ("scan1",))
assert cur.fetchone()[0] == 2
cur.execute('SELECT count(*) FROM "LibraryItem"')
assert cur.fetchone()[0] == 2
def test_scan_chunk_resumes_from_cursor(conn, tmp_path):
def test_scan_chunk_resumes_and_finishes(conn, tmp_path):
resolver, browser = _tree(tmp_path)
cursor, imported, skipped, done = scan_chunk(
conn, resolver, FakeProbe(quality_class=2), browser, str(tmp_path),
cursor="Artist A/Beta (2002)", limit=25)
assert (imported, skipped, done) == (1, 0, True) # only Gamma remains
assert cursor == "Artist B/Gamma (2003)"
build_worklist(conn, "scan1", str(tmp_path))
scan_chunk(conn, resolver, FakeProbe(quality_class=2), browser, "scan1", limit=2)
imported, skipped, done = scan_chunk(conn, resolver, FakeProbe(quality_class=2), browser, "scan1", limit=25)
assert (imported, skipped, done) == (1, 0, True) # last album, then drained
with conn.cursor() as cur:
cur.execute('SELECT album FROM "LibraryItem"')
assert cur.fetchall() == [("Gamma",)] # only Gamma imported
cur.execute('SELECT album FROM "LibraryItem" ORDER BY album')
assert cur.fetchall() == [("Alpha",), ("Beta",), ("Gamma",)]
def test_scan_chunk_cursor_past_end_is_noop(conn, tmp_path):
def test_scan_chunk_one_per_chunk_totals_correctly(conn, tmp_path):
resolver, browser = _tree(tmp_path)
cursor, imported, skipped, done = scan_chunk(
conn, resolver, FakeProbe(), browser, str(tmp_path), cursor="zzzz/zzzz", limit=10)
assert (imported, skipped, done) == (0, 0, True)
assert cursor == "zzzz/zzzz" # unchanged
assert _counts(conn)["LibraryItem"] == 0
def test_scan_chunk_splits_artist_across_chunks_idempotently(conn, tmp_path):
# limit=1 forces Artist A's two albums into separate chunks, re-browsing its
# discography — which must stay idempotent (ON CONFLICT) and total the same as one scan.
resolver, browser = _tree(tmp_path)
cursor, done, total_i, total_s, guard = "", False, 0, 0, 0
build_worklist(conn, "scan1", str(tmp_path))
total_i = total_s = guard = 0
done = False
while not done and guard < 10:
cursor, i, s, done = scan_chunk(
conn, resolver, FakeProbe(quality_class=2), browser, str(tmp_path), cursor=cursor, limit=1)
i, s, done = scan_chunk(conn, resolver, FakeProbe(quality_class=2), browser, "scan1", limit=1)
total_i += i
total_s += s
guard += 1
assert (total_i, total_s) == (3, 0)
assert guard == 3 # three albums, one per chunk, then done
assert guard == 3
assert _counts(conn) == {"LibraryItem": 3, "MonitoredRelease": 3, "WatchedArtist": 2}
def test_scan_chunk_unbounded_matches_full_walk(conn, tmp_path):
resolver, browser = _tree(tmp_path)
cursor, imported, skipped, done = scan_chunk(
conn, resolver, FakeProbe(quality_class=2), browser, str(tmp_path), cursor="", limit=None)
assert (imported, skipped, done) == (3, 0, True)
assert cursor == "Artist B/Gamma (2003)"
assert _counts(conn) == {"LibraryItem": 3, "MonitoredRelease": 3, "WatchedArtist": 2}
def test_clear_worklist_removes_this_scans_rows(conn, tmp_path):
resolver, _ = _tree(tmp_path)
build_worklist(conn, "scan1", str(tmp_path))
clear_worklist(conn, "scan1")
with conn.cursor() as cur:
cur.execute('SELECT count(*) FROM "ScanWorkItem" WHERE "scanId" = %s', ("scan1",))
assert cur.fetchone()[0] == 0