From dcdb9752a245aae47ddd7517ea06af7cb11569b9 Mon Sep 17 00:00:00 2001 From: Jonathan Date: Fri, 10 Jul 2026 16:23:45 +0200 Subject: [PATCH] Add walking-skeleton implementation plan (slice 1, plan 1) Seven TDD tasks: monorepo + compose scaffold, Prisma Request/Job schema, request API, live queue UI, worker claim logic, fake pipeline, and worker loop + Dockerfiles with end-to-end verification. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-07-10-lyra-walking-skeleton.md | 1086 +++++++++++++++++ 1 file changed, 1086 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-10-lyra-walking-skeleton.md diff --git a/docs/superpowers/plans/2026-07-10-lyra-walking-skeleton.md b/docs/superpowers/plans/2026-07-10-lyra-walking-skeleton.md new file mode 100644 index 0000000..01c82e4 --- /dev/null +++ b/docs/superpowers/plans/2026-07-10-lyra-walking-skeleton.md @@ -0,0 +1,1086 @@ +# Lyra Walking Skeleton Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Stand up the end-to-end Lyra architecture — a Next.js UI/API, a Python worker, and Postgres — where a user submits an album request, the worker claims the resulting job and drives it through the full state machine via a *fake* pipeline, and the UI shows live progress. Zero source integration. + +**Architecture:** Three Docker Compose services — `web` (Next.js App Router, owns the Prisma schema), `worker` (Python poll-loop service), and `db` (Postgres). The web app writes `Request` + `Job` rows; the worker claims jobs with `SELECT … FOR UPDATE SKIP LOCKED` and advances each through the state machine. Communication is entirely through Postgres, exactly as slices 2–4 will use it. + +**Tech Stack:** Next.js 15 (App Router, TypeScript, React 19), Prisma 6, PostgreSQL 17, Python 3.12 with psycopg 3. Tests: Vitest 2 (web API), pytest 8 (worker). + +## Global Constraints + +- Node.js 22 LTS; Next.js App Router (not Pages Router); TypeScript strict mode on. +- Python 3.12; database access via `psycopg` v3 (not psycopg2); raw SQL only in the worker. +- **Prisma is the single owner of the database schema.** The worker never issues DDL; it reads/writes tables Prisma migrated. Worker tests assume `prisma migrate deploy` has already been run against the target database. +- PostgreSQL 17. +- One `DATABASE_URL` (Postgres connection string) is shared by both services via environment. +- Job state values (exact): `requested`, `matching`, `matched`, `downloading`, `tagging`, `imported`, `needs_attention`. +- Request status values (exact): `pending`, `completed`, `needs_attention`. +- Monorepo layout: `web/` (Next.js), `worker/` (Python), root holds `docker-compose.yml` and `.env.example`. +- Every task ends with a commit. + +--- + +### Task 1: Monorepo scaffold + Docker Compose + +**Files:** +- Create: `web/package.json` +- Create: `web/tsconfig.json` +- Create: `web/next.config.ts` +- Create: `web/src/app/layout.tsx` +- Create: `web/src/app/page.tsx` +- Create: `worker/requirements.txt` +- Create: `worker/lyra_worker/__init__.py` +- Create: `worker/tests/__init__.py` +- Create: `worker/pytest.ini` +- Create: `docker-compose.yml` +- Create: `.env.example` + +**Interfaces:** +- Consumes: nothing (first task). +- Produces: a runnable Next.js app at `web/`, an importable Python package `lyra_worker`, and a Compose stack with services `web`, `worker`, `db` sharing `DATABASE_URL`. + +- [ ] **Step 1: Create the Next.js app manifest and config** + +`web/package.json`: +```json +{ + "name": "lyra-web", + "version": "0.1.0", + "private": true, + "scripts": { + "dev": "next dev", + "build": "next build", + "start": "next start", + "test": "vitest run" + }, + "dependencies": { + "next": "15.1.0", + "react": "19.0.0", + "react-dom": "19.0.0", + "@prisma/client": "6.1.0" + }, + "devDependencies": { + "typescript": "5.7.2", + "@types/node": "22.10.2", + "@types/react": "19.0.1", + "@types/react-dom": "19.0.1", + "prisma": "6.1.0", + "vitest": "2.1.8" + } +} +``` + +`web/tsconfig.json`: +```json +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["dom", "dom.iterable", "ES2022"], + "allowJs": true, + "skipLibCheck": true, + "strict": true, + "noEmit": true, + "esModuleInterop": true, + "module": "esnext", + "moduleResolution": "bundler", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "incremental": true, + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./src/*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} +``` + +`web/next.config.ts`: +```ts +import type { NextConfig } from "next"; + +const nextConfig: NextConfig = { + output: "standalone", +}; + +export default nextConfig; +``` + +- [ ] **Step 2: Create a minimal placeholder UI so the app builds** + +`web/src/app/layout.tsx`: +```tsx +export const metadata = { title: "Lyra" }; + +export default function RootLayout({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} +``` + +`web/src/app/page.tsx`: +```tsx +export default function Home() { + return
Lyra
; +} +``` + +- [ ] **Step 3: Create the Python worker package skeleton** + +`worker/requirements.txt`: +``` +psycopg[binary]==3.2.3 +pytest==8.3.4 +``` + +`worker/lyra_worker/__init__.py`: +```python +"""Lyra acquisition worker.""" + +__version__ = "0.1.0" +``` + +`worker/tests/__init__.py`: +```python +``` + +`worker/pytest.ini`: +```ini +[pytest] +testpaths = tests +``` + +- [ ] **Step 4: Create the Compose stack and env template** + +`.env.example`: +``` +POSTGRES_USER=lyra +POSTGRES_PASSWORD=lyra +POSTGRES_DB=lyra +DATABASE_URL=postgresql://lyra:lyra@db:5432/lyra +``` + +`docker-compose.yml`: +```yaml +services: + db: + image: postgres:17 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 5 + + web: + build: ./web + environment: + DATABASE_URL: ${DATABASE_URL} + ports: + - "3000:3000" + depends_on: + db: + condition: service_healthy + + worker: + build: ./worker + environment: + DATABASE_URL: ${DATABASE_URL} + depends_on: + db: + condition: service_healthy + +volumes: + postgres-data: +``` + +- [ ] **Step 5: Verify the scaffold builds** + +Run: +```bash +cp .env.example .env +cd web && npm install && npm run build +cd ../worker && python -m pip install -r requirements.txt && python -c "import lyra_worker; print(lyra_worker.__version__)" +``` +Expected: `next build` completes without errors; the Python import prints `0.1.0`. + +- [ ] **Step 6: Verify Compose config is valid** + +Run: `docker compose config -q && echo OK` +Expected: prints `OK` (no schema errors). + +- [ ] **Step 7: Commit** + +```bash +git add web worker docker-compose.yml .env.example +git commit -m "chore: scaffold web, worker, and compose stack" +``` + +--- + +### Task 2: Prisma schema + migration for Request and Job + +**Files:** +- Create: `web/prisma/schema.prisma` +- Create: `web/src/lib/db.ts` +- Modify: `web/package.json` (add `postinstall` + `db:migrate` scripts) + +**Interfaces:** +- Consumes: `DATABASE_URL` from Task 1. +- Produces: + - Tables `Request(id, artist, album, status, createdAt)` and `Job(id, requestId, state, currentStage, attempts, error, claimedAt, createdAt, updatedAt)`. + - Enums `RequestStatus { pending, completed, needs_attention }` and `JobState { requested, matching, matched, downloading, tagging, imported, needs_attention }`. + - A Prisma client singleton exported as `prisma` from `@/lib/db`. + +- [ ] **Step 1: Write the Prisma schema** + +`web/prisma/schema.prisma`: +```prisma +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} + +enum RequestStatus { + pending + completed + needs_attention +} + +enum JobState { + requested + matching + matched + downloading + tagging + imported + needs_attention +} + +model Request { + id String @id @default(cuid()) + artist String + album String + status RequestStatus @default(pending) + createdAt DateTime @default(now()) + job Job? +} + +model Job { + id String @id @default(cuid()) + request Request @relation(fields: [requestId], references: [id], onDelete: Cascade) + requestId String @unique + state JobState @default(requested) + currentStage String @default("intake") + attempts Int @default(0) + error String? + claimedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt +} +``` + +- [ ] **Step 2: Add the Prisma client singleton** + +`web/src/lib/db.ts`: +```ts +import { PrismaClient } from "@prisma/client"; + +const globalForPrisma = globalThis as unknown as { prisma?: PrismaClient }; + +export const prisma = globalForPrisma.prisma ?? new PrismaClient(); + +if (process.env.NODE_ENV !== "production") globalForPrisma.prisma = prisma; +``` + +- [ ] **Step 3: Add migration/generate scripts to package.json** + +In `web/package.json`, add to `"scripts"`: +```json + "postinstall": "prisma generate", + "db:migrate": "prisma migrate deploy", + "db:dev": "prisma migrate dev" +``` + +- [ ] **Step 4: Create the migration and apply it** + +Run (Postgres must be up — `docker compose up -d db`): +```bash +cd web +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +npx prisma migrate dev --name init_request_job +``` +Expected: a migration is created under `web/prisma/migrations/` and applied; Prisma prints "Your database is now in sync with your schema." + +- [ ] **Step 5: Verify the tables exist** + +Run: +```bash +cd web +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +npx prisma db execute --stdin <<'SQL' +SELECT 1 FROM "Request" LIMIT 0; +SELECT 1 FROM "Job" LIMIT 0; +SQL +echo "tables OK" +``` +Expected: no error; prints `tables OK`. + +- [ ] **Step 6: Commit** + +```bash +git add web/prisma web/src/lib/db.ts web/package.json +git commit -m "feat: add Request and Job schema with migration" +``` + +--- + +### Task 3: Request API (create + list) + +**Files:** +- Create: `web/src/app/api/requests/route.ts` +- Create: `web/vitest.config.ts` +- Create: `web/src/test/setup.ts` +- Test: `web/src/app/api/requests/route.test.ts` +- Modify: `web/package.json` (add `@vitejs/plugin-react` not needed; Vitest config uses node env) + +**Interfaces:** +- Consumes: `prisma` from `@/lib/db` (Task 2). +- Produces: + - `POST /api/requests` — body `{ artist: string, album: string }` → creates a `Request` (status `pending`) and its `Job` (state `requested`) in one transaction; responds `201` with `{ id, artist, album, status, job: { id, state, currentStage } }`. + - `GET /api/requests` — responds `200` with `{ requests: Array<{ id, artist, album, status, createdAt, job: { state, currentStage } | null }> }`, newest first. + +- [ ] **Step 1: Write the failing test** + +`web/vitest.config.ts`: +```ts +import { defineConfig } from "vitest/config"; +import { fileURLToPath } from "node:url"; + +export default defineConfig({ + resolve: { + alias: { + "@": fileURLToPath(new URL("./src", import.meta.url)), + }, + }, + test: { + environment: "node", + setupFiles: ["./src/test/setup.ts"], + fileParallelism: false, + }, +}); +``` + +`web/src/test/setup.ts`: +```ts +import { beforeEach } from "vitest"; +import { prisma } from "@/lib/db"; + +beforeEach(async () => { + // Job has a cascade FK to Request; delete Job first. + await prisma.job.deleteMany(); + await prisma.request.deleteMany(); +}); +``` + +`web/src/app/api/requests/route.test.ts`: +```ts +import { describe, it, expect } from "vitest"; +import { POST, GET } from "./route"; + +function postReq(body: unknown) { + return new Request("http://localhost/api/requests", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +describe("requests API", () => { + it("creates a request with an initial job", async () => { + const res = await POST(postReq({ artist: "Radiohead", album: "In Rainbows" })); + expect(res.status).toBe(201); + const body = await res.json(); + expect(body.artist).toBe("Radiohead"); + expect(body.status).toBe("pending"); + expect(body.job.state).toBe("requested"); + expect(body.job.currentStage).toBe("intake"); + }); + + it("rejects a request missing fields", async () => { + const res = await POST(postReq({ artist: "Radiohead" })); + expect(res.status).toBe(400); + }); + + it("lists requests newest first with job state", async () => { + await POST(postReq({ artist: "A", album: "1" })); + await POST(postReq({ artist: "B", album: "2" })); + const res = await GET(); + expect(res.status).toBe(200); + const body = await res.json(); + expect(body.requests).toHaveLength(2); + expect(body.requests[0].artist).toBe("B"); + expect(body.requests[0].job.state).toBe("requested"); + }); +}); +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: +```bash +cd web +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +npx vitest run src/app/api/requests/route.test.ts +``` +Expected: FAIL — cannot import `./route` (module does not exist). + +- [ ] **Step 3: Implement the route handlers** + +`web/src/app/api/requests/route.ts`: +```ts +import { prisma } from "@/lib/db"; + +export async function POST(request: Request) { + let body: unknown; + try { + body = await request.json(); + } catch { + return Response.json({ error: "invalid JSON" }, { status: 400 }); + } + + const { artist, album } = (body ?? {}) as { artist?: unknown; album?: unknown }; + if (typeof artist !== "string" || !artist.trim() || typeof album !== "string" || !album.trim()) { + return Response.json({ error: "artist and album are required" }, { status: 400 }); + } + + const created = await prisma.request.create({ + data: { + artist: artist.trim(), + album: album.trim(), + job: { create: {} }, + }, + include: { job: true }, + }); + + return Response.json( + { + id: created.id, + artist: created.artist, + album: created.album, + status: created.status, + job: { id: created.job!.id, state: created.job!.state, currentStage: created.job!.currentStage }, + }, + { status: 201 }, + ); +} + +export async function GET() { + const requests = await prisma.request.findMany({ + orderBy: { createdAt: "desc" }, + include: { job: true }, + }); + + return Response.json({ + requests: requests.map((r) => ({ + id: r.id, + artist: r.artist, + album: r.album, + status: r.status, + createdAt: r.createdAt, + job: r.job ? { state: r.job.state, currentStage: r.job.currentStage } : null, + })), + }); +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: +```bash +cd web +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +npx vitest run src/app/api/requests/route.test.ts +``` +Expected: PASS — all three tests green. + +- [ ] **Step 5: Commit** + +```bash +git add web/src/app/api web/vitest.config.ts web/src/test/setup.ts +git commit -m "feat: add request create/list API" +``` + +--- + +### Task 4: Request form + live queue UI + +**Files:** +- Modify: `web/src/app/page.tsx` +- Create: `web/src/app/queue.tsx` + +**Interfaces:** +- Consumes: `POST /api/requests` and `GET /api/requests` (Task 3). +- Produces: a client page that submits a new request and polls the queue every 2 seconds, rendering each request's artist/album and live job state. + +- [ ] **Step 1: Write the client queue component** + +`web/src/app/queue.tsx`: +```tsx +"use client"; + +import { useEffect, useState } from "react"; + +type Row = { + id: string; + artist: string; + album: string; + status: string; + job: { state: string; currentStage: string } | null; +}; + +export function Queue() { + const [rows, setRows] = useState([]); + const [artist, setArtist] = useState(""); + const [album, setAlbum] = useState(""); + + async function refresh() { + const res = await fetch("/api/requests"); + const body = await res.json(); + setRows(body.requests); + } + + useEffect(() => { + refresh(); + const id = setInterval(refresh, 2000); + return () => clearInterval(id); + }, []); + + async function submit(e: React.FormEvent) { + e.preventDefault(); + if (!artist.trim() || !album.trim()) return; + await fetch("/api/requests", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ artist, album }), + }); + setArtist(""); + setAlbum(""); + refresh(); + } + + return ( +
+
+ setArtist(e.target.value)} /> + setAlbum(e.target.value)} /> + +
+
    + {rows.map((r) => ( +
  • + {r.artist} — {r.album} · {r.job?.state ?? r.status} + {r.job ? ` (${r.job.currentStage})` : ""} +
    • + ))} +
+
+ ); +} +``` + +- [ ] **Step 2: Render the queue on the home page** + +`web/src/app/page.tsx`: +```tsx +import { Queue } from "./queue"; + +export default function Home() { + return ( +
+

Lyra

+ +
+ ); +} +``` + +- [ ] **Step 3: Verify the app builds** + +Run: `cd web && npm run build` +Expected: build completes; `/` is compiled as a route with no type errors. + +- [ ] **Step 4: Commit** + +```bash +git add web/src/app/page.tsx web/src/app/queue.tsx +git commit -m "feat: add request form and live queue UI" +``` + +--- + +### Task 5: Worker job-claim logic + +**Files:** +- Create: `worker/lyra_worker/db.py` +- Create: `worker/lyra_worker/claim.py` +- Create: `worker/tests/conftest.py` +- Test: `worker/tests/test_claim.py` + +**Interfaces:** +- Consumes: `DATABASE_URL`; the `Job`/`Request` tables from Task 2. +- Produces: + - `connect() -> psycopg.Connection` in `db.py`. + - `claim_next(conn) -> str | None` in `claim.py`: atomically selects the oldest `Job` in state `requested` using `FOR UPDATE SKIP LOCKED`, sets its state to `matching`, `claimedAt = now()`, `currentStage = 'match'`, increments `attempts`, commits, and returns the job id (or `None` if none available). + +- [ ] **Step 1: Add the DB connection helper** + +`worker/lyra_worker/db.py`: +```python +import os + +import psycopg + + +def connect() -> psycopg.Connection: + dsn = os.environ["DATABASE_URL"] + return psycopg.connect(dsn) +``` + +- [ ] **Step 2: Write the test fixtures** + +`worker/tests/conftest.py`: +```python +import os + +import psycopg +import pytest + + +@pytest.fixture() +def conn(): + dsn = os.environ["DATABASE_URL"] + connection = psycopg.connect(dsn) + yield connection + # Clean up rows created during the test (Job cascades from Request). + with connection.cursor() as cur: + cur.execute('DELETE FROM "Job"') + cur.execute('DELETE FROM "Request"') + connection.commit() + connection.close() + + +def insert_request(conn, artist="Artist", album="Album"): + """Insert a Request + its Job (state 'requested') and return the job id.""" + with conn.cursor() as cur: + cur.execute( + 'INSERT INTO "Request" (id, artist, album, status, "createdAt") ' + "VALUES (gen_random_uuid()::text, %s, %s, 'pending', now()) RETURNING id", + (artist, album), + ) + request_id = cur.fetchone()[0] + cur.execute( + 'INSERT INTO "Job" (id, "requestId", state, "currentStage", attempts, "createdAt", "updatedAt") ' + "VALUES (gen_random_uuid()::text, %s, 'requested', 'intake', 0, now(), now()) RETURNING id", + (request_id,), + ) + job_id = cur.fetchone()[0] + conn.commit() + return job_id +``` + +- [ ] **Step 3: Write the failing test** + +`worker/tests/test_claim.py`: +```python +from lyra_worker.claim import claim_next +from tests.conftest import insert_request + + +def test_claim_returns_none_when_empty(conn): + assert claim_next(conn) is None + + +def test_claim_advances_job_to_matching(conn): + job_id = insert_request(conn) + claimed = claim_next(conn) + assert claimed == job_id + + with conn.cursor() as cur: + cur.execute('SELECT state, "currentStage", attempts, "claimedAt" FROM "Job" WHERE id = %s', (job_id,)) + state, stage, attempts, claimed_at = cur.fetchone() + assert state == "matching" + assert stage == "match" + assert attempts == 1 + assert claimed_at is not None + + +def test_claim_does_not_repick_claimed_job(conn): + insert_request(conn) + assert claim_next(conn) is not None + assert claim_next(conn) is None +``` + +- [ ] **Step 4: Run the test to verify it fails** + +Run: +```bash +cd worker +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +python -m pytest tests/test_claim.py -v +``` +Expected: FAIL — `ModuleNotFoundError: No module named 'lyra_worker.claim'`. + +- [ ] **Step 5: Implement claim_next** + +`worker/lyra_worker/claim.py`: +```python +import psycopg + + +def claim_next(conn: psycopg.Connection) -> str | None: + """Atomically claim the oldest queued job. Returns its id or None.""" + with conn.cursor() as cur: + cur.execute( + """ + SELECT id FROM "Job" + WHERE state = 'requested' + ORDER BY "createdAt" ASC + FOR UPDATE SKIP LOCKED + LIMIT 1 + """ + ) + row = cur.fetchone() + if row is None: + conn.rollback() + return None + job_id = row[0] + cur.execute( + """ + UPDATE "Job" + SET state = 'matching', + "currentStage" = 'match', + "claimedAt" = now(), + attempts = attempts + 1, + "updatedAt" = now() + WHERE id = %s + """, + (job_id,), + ) + conn.commit() + return job_id +``` + +- [ ] **Step 6: Run the test to verify it passes** + +Run: +```bash +cd worker +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +python -m pytest tests/test_claim.py -v +``` +Expected: PASS — all three tests green. + +- [ ] **Step 7: Commit** + +```bash +git add worker/lyra_worker/db.py worker/lyra_worker/claim.py worker/tests/conftest.py worker/tests/test_claim.py +git commit -m "feat: add worker job-claim logic" +``` + +--- + +### Task 6: Fake pipeline through the state machine + +**Files:** +- Create: `worker/lyra_worker/pipeline.py` +- Test: `worker/tests/test_pipeline.py` + +**Interfaces:** +- Consumes: a claimed job id (state `matching`) from Task 5; the DB connection. +- Produces: `run_pipeline(conn, job_id, stage_delay=0.0) -> None`: advances the job through `matched → downloading → tagging → imported`, updating `state` and `currentStage` at each step, then sets the parent `Request.status` to `completed`. `stage_delay` seconds are slept between stages (default `0.0` for tests; the runtime loop passes a small value so progress is visible in the UI). + +- [ ] **Step 1: Write the failing test** + +`worker/tests/test_pipeline.py`: +```python +from lyra_worker.claim import claim_next +from lyra_worker.pipeline import run_pipeline +from tests.conftest import insert_request + + +def test_pipeline_drives_job_to_imported(conn): + job_id = insert_request(conn) + claim_next(conn) + + run_pipeline(conn, job_id, stage_delay=0.0) + + with conn.cursor() as cur: + cur.execute('SELECT state, "currentStage" FROM "Job" WHERE id = %s', (job_id,)) + state, stage = cur.fetchone() + cur.execute('SELECT status FROM "Request" WHERE id = (SELECT "requestId" FROM "Job" WHERE id = %s)', (job_id,)) + req_status = cur.fetchone()[0] + + assert state == "imported" + assert stage == "import" + assert req_status == "completed" +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: +```bash +cd worker +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +python -m pytest tests/test_pipeline.py -v +``` +Expected: FAIL — `ModuleNotFoundError: No module named 'lyra_worker.pipeline'`. + +- [ ] **Step 3: Implement the fake pipeline** + +`worker/lyra_worker/pipeline.py`: +```python +import time + +import psycopg + +# (state, currentStage) pairs the job moves through, in order, after claim. +_STAGES = [ + ("matched", "rank"), + ("downloading", "download"), + ("tagging", "tag"), + ("imported", "import"), +] + + +def _set_state(conn: psycopg.Connection, job_id: str, state: str, stage: str) -> None: + with conn.cursor() as cur: + cur.execute( + 'UPDATE "Job" SET state = %s, "currentStage" = %s, "updatedAt" = now() WHERE id = %s', + (state, stage, job_id), + ) + conn.commit() + + +def run_pipeline(conn: psycopg.Connection, job_id: str, stage_delay: float = 0.0) -> None: + """Fake acquisition: walk the claimed job through to 'imported'.""" + for state, stage in _STAGES: + if stage_delay: + time.sleep(stage_delay) + _set_state(conn, job_id, state, stage) + + with conn.cursor() as cur: + cur.execute( + 'UPDATE "Request" SET status = \'completed\' ' + 'WHERE id = (SELECT "requestId" FROM "Job" WHERE id = %s)', + (job_id,), + ) + conn.commit() +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: +```bash +cd worker +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +python -m pytest tests/test_pipeline.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Run the whole worker suite** + +Run: +```bash +cd worker +export DATABASE_URL=postgresql://lyra:lyra@localhost:5432/lyra +python -m pytest -v +``` +Expected: PASS — claim and pipeline tests all green. + +- [ ] **Step 6: Commit** + +```bash +git add worker/lyra_worker/pipeline.py worker/tests/test_pipeline.py +git commit -m "feat: add fake acquisition pipeline" +``` + +--- + +### Task 7: Worker main loop + Dockerfiles + end-to-end verification + +**Files:** +- Create: `worker/lyra_worker/main.py` +- Create: `worker/Dockerfile` +- Create: `web/Dockerfile` +- Create: `web/.dockerignore` +- Create: `worker/.dockerignore` +- Create: `web/docker-entrypoint.sh` + +**Interfaces:** +- Consumes: `claim_next` (Task 5), `run_pipeline` (Task 6), `connect` (Task 5). +- Produces: a long-running worker process (`python -m lyra_worker.main`) that repeatedly claims a job and runs the pipeline, sleeping when idle; production Dockerfiles for both services; the web container runs `prisma migrate deploy` before starting. + +- [ ] **Step 1: Implement the worker main loop** + +`worker/lyra_worker/main.py`: +```python +import time + +from lyra_worker.claim import claim_next +from lyra_worker.db import connect +from lyra_worker.pipeline import run_pipeline + +IDLE_SLEEP = 2.0 +STAGE_DELAY = 1.0 # visible progress in the UI + + +def run_forever() -> None: + conn = connect() + print("worker: waiting for jobs", flush=True) + try: + while True: + job_id = claim_next(conn) + if job_id is None: + time.sleep(IDLE_SLEEP) + continue + print(f"worker: claimed job {job_id}", flush=True) + run_pipeline(conn, job_id, stage_delay=STAGE_DELAY) + print(f"worker: imported job {job_id}", flush=True) + finally: + conn.close() + + +if __name__ == "__main__": + run_forever() +``` + +- [ ] **Step 2: Add the worker Dockerfile and dockerignore** + +`worker/Dockerfile`: +```dockerfile +FROM python:3.12-slim +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt +COPY lyra_worker ./lyra_worker +CMD ["python", "-m", "lyra_worker.main"] +``` + +`worker/.dockerignore`: +``` +tests +__pycache__ +*.pyc +``` + +- [ ] **Step 3: Create the empty public dir and web dockerignore** + +Create `web/public/.gitkeep` (empty file — the standalone image copies `public/`, which must exist): +``` +``` + +`web/.dockerignore`: +``` +node_modules +.next +.env +``` + +- [ ] **Step 4: Add the web Dockerfile and entrypoint** + +`web/docker-entrypoint.sh`: +```bash +#!/bin/sh +set -e +npx prisma migrate deploy +exec node server.js +``` + +`web/Dockerfile`: +```dockerfile +FROM node:22-slim AS build +WORKDIR /app +RUN apt-get update && apt-get install -y openssl && rm -rf /var/lib/apt/lists/* +COPY package.json package-lock.json* ./ +RUN npm install +COPY . . +RUN npx prisma generate && npm run build + +FROM node:22-slim AS run +WORKDIR /app +RUN apt-get update && apt-get install -y openssl && rm -rf /var/lib/apt/lists/* +ENV NODE_ENV=production +COPY --from=build /app/.next/standalone ./ +COPY --from=build /app/.next/static ./.next/static +COPY --from=build /app/public ./public +COPY --from=build /app/prisma ./prisma +COPY --from=build /app/node_modules/.bin/prisma ./node_modules/.bin/prisma +COPY --from=build /app/node_modules/prisma ./node_modules/prisma +COPY --from=build /app/node_modules/@prisma ./node_modules/@prisma +COPY --from=build /app/docker-entrypoint.sh ./ +RUN chmod +x docker-entrypoint.sh +EXPOSE 3000 +CMD ["./docker-entrypoint.sh"] +``` + +- [ ] **Step 5: Build and start the whole stack** + +Run: +```bash +docker compose build +docker compose up -d +sleep 15 +docker compose logs worker | tail -n 5 +``` +Expected: worker logs show `worker: waiting for jobs`; web and db containers are healthy. + +- [ ] **Step 6: Verify end-to-end via the API** + +Run: +```bash +curl -s -X POST http://localhost:3000/api/requests \ + -H "content-type: application/json" \ + -d '{"artist":"Radiohead","album":"In Rainbows"}' >/dev/null +# Poll until the request completes (worker takes ~4s with STAGE_DELAY=1s). +for i in $(seq 1 15); do + sleep 2 + state=$(curl -s http://localhost:3000/api/requests | python3 -c 'import sys,json;print(json.load(sys.stdin)["requests"][0]["job"]["state"])') + echo "state=$state" + [ "$state" = "imported" ] && break +done +``` +Expected: the printed state advances through `matching`/`downloading`/etc. and reaches `imported`. + +- [ ] **Step 7: Tear down** + +Run: `docker compose down` +Expected: all containers stopped and removed. + +- [ ] **Step 8: Commit** + +```bash +git add worker/lyra_worker/main.py worker/Dockerfile worker/.dockerignore \ + web/Dockerfile web/.dockerignore web/docker-entrypoint.sh web/public/.gitkeep +git commit -m "feat: add worker loop and production Dockerfiles" +``` + +--- + +## Notes for the next plan (Pipeline core) + +The fake `run_pipeline` in `worker/lyra_worker/pipeline.py` is the seam the next plan replaces. Plan 2 introduces the real six-stage pipeline, the `SourceAdapter` contract, the ranker + confidence scoring, and fake adapters — swapping the fake stage-walk for real stage functions while keeping the claim loop, state machine, and DB schema unchanged. The `candidates`, `releases`, `artists`, `library_items`, and `config` tables from the design spec are added by their respective plans as the stages that need them arrive.