Jonathan/Lyra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(scan): extract scan_chunk from scan_library

Browse Source 
scan_chunk resolves album folders whose (artist, album) sorts after a
cursor, up to a limit, returning (new_cursor, imported, skipped, done).
scan_library becomes the unbounded case (limit=None), so its behavior and
tests are unchanged. Idempotent: all writes are ON CONFLICT, so an
overlapping/repeated cursor is safe. Enables chunked scanning in the loop.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jonathan
2026-07-12 12:36:54 +02:00
parent ade9cd7694
commit 62b60e3dae
2 changed files with 154 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
worker/lyra_worker/scan.py
+73 -27
View File
@@ -91,16 +91,11 @@ def _record_owned(conn, target, quality_class: int, fmt: str, path: str, watched
return newly
def scan_library(conn: psycopg.Connection, resolver, probe: AudioProbe, browser, dest_root: str = "/music") -> ScanResult:
"""Walk `{dest_root}/{Artist}/{Album}/`, resolve each album against MusicBrainz, and record
matched albums as have + monitored + followed. Also browses each followed artist's full
discography (once per scan run) and stores it as unmonitored releases, matching web-follow
behavior. Unmatched or audio-less folders are skipped."""
imported = 0
skipped = 0
browsed: set[str] = set() # artists whose discography we've populated this run
def _iter_album_entries(dest_root: str):
"""Yield (artist_name, album_folder, album_path) for each candidate album directory,
in the stable nested-sorted order the scan cursor relies on."""
if not os.path.isdir(dest_root):
return ScanResult(0, 0)
return
for artist_name in sorted(os.listdir(dest_root)):
artist_path = os.path.join(dest_root, artist_name)
if artist_name.startswith(".") or not os.path.isdir(artist_path):
@@ -109,22 +104,73 @@ def scan_library(conn: psycopg.Connection, resolver, probe: AudioProbe, browser,
album_path = os.path.join(artist_path, album_folder)
if album_folder.startswith(".") or not os.path.isdir(album_path):
continue
audio = _first_audio(album_path)
if audio is None:
continue # no audio → not an album
pr = probe.probe(audio)
album_name = _parse_album_dir(album_folder)
target = resolver.resolve(artist_name, album_name) if artist_name and album_name else None
if target is None: # fall back to the file's embedded tags
if pr.artist and pr.album and (pr.artist, pr.album) != (artist_name, album_name):
target = resolver.resolve(pr.artist, pr.album)
if target is None:
skipped += 1
continue
watched_id = _ensure_artist(conn, target)
if watched_id is not None and target.artist_mbid not in browsed:
_populate_discography(conn, browser, watched_id, target.artist_mbid, target.artist)
browsed.add(target.artist_mbid)
if _record_owned(conn, target, pr.quality_class, pr.fmt, album_path, watched_id):
imported += 1
yield artist_name, album_folder, album_path
def _cursor_key(artist_name: str, album_folder: str) -> str:
return f"{artist_name}/{album_folder}"
def _cursor_tuple(cursor: str) -> tuple[str, str]:
# A cursor is "<artist>/<album>"; artist and album are single path segments (no "/"),
# so partitioning on the first "/" recovers them, and tuple order matches the walk.
artist, _, album = (cursor or "").partition("/")
return artist, album
def _process_album(conn, resolver, probe, browser, artist_name, album_folder, album_path, browsed) -> str:
"""Identify and record one album folder. Returns 'imported' (newly recorded),
'recorded' (matched but already present), 'skipped' (no MB match) or 'none' (no audio)."""
audio = _first_audio(album_path)
if audio is None:
return "none" # no audio → not an album
pr = probe.probe(audio)
album_name = _parse_album_dir(album_folder)
target = resolver.resolve(artist_name, album_name) if artist_name and album_name else None
if target is None: # fall back to the file's embedded tags
if pr.artist and pr.album and (pr.artist, pr.album) != (artist_name, album_name):
target = resolver.resolve(pr.artist, pr.album)
if target is None:
return "skipped"
watched_id = _ensure_artist(conn, target)
if watched_id is not None and target.artist_mbid not in browsed:
_populate_discography(conn, browser, watched_id, target.artist_mbid, target.artist)
browsed.add(target.artist_mbid)
return "imported" if _record_owned(conn, target, pr.quality_class, pr.fmt, album_path, watched_id) else "recorded"
def scan_chunk(conn: psycopg.Connection, resolver, probe: AudioProbe, browser, dest_root: str = "/music",
cursor: str = "", limit: int | None = None) -> tuple[str, int, int, bool]:
"""Resolve and record album folders whose (artist, album) sorts after `cursor`, up to
`limit` of them (unbounded when limit is None). Returns (new_cursor, imported, skipped,
done); `done` is True when the walk was exhausted within this chunk. All writes are
ON CONFLICT, so an overlapping or repeated cursor is idempotent."""
after = _cursor_tuple(cursor)
imported = skipped = processed = 0
browsed: set[str] = set() # artists whose discography we've populated this chunk
new_cursor = cursor
done = True
for artist_name, album_folder, album_path in _iter_album_entries(dest_root):
if (artist_name, album_folder) <= after:
continue # already processed by an earlier chunk
if limit is not None and processed >= limit:
done = False # stopped early; more albums remain after new_cursor
break
outcome = _process_album(conn, resolver, probe, browser, artist_name, album_folder, album_path, browsed)
if outcome == "imported":
imported += 1
elif outcome == "skipped":
skipped += 1
new_cursor = _cursor_key(artist_name, album_folder)
processed += 1
return new_cursor, imported, skipped, done
def scan_library(conn: psycopg.Connection, resolver, probe: AudioProbe, browser, dest_root: str = "/music") -> ScanResult:
"""Walk `{dest_root}/{Artist}/{Album}/` in one unbounded pass, resolving each album
against MusicBrainz and recording matched albums as have + monitored + followed (plus
each followed artist's full discography as unmonitored releases). Unmatched or audio-less
folders are skipped. This is `scan_chunk` with no limit; the worker loop uses the chunked
form so a large library doesn't block job-claim."""
_cursor, imported, skipped, _done = scan_chunk(conn, resolver, probe, browser, dest_root, cursor="", limit=None)
return ScanResult(imported, skipped)