Stop the candidate ladder from hammering slskd through a stall

A live request for "Michael Jackson - Chicago" exercised both new paths
correctly -- the blocked artist name fell back to the bare title and found real
Xscape rips, and the ladder tried five peers instead of dying on the first --
and then failed anyway, with all five reporting a 500 from slskd.

slskd's own log shows why, and it is not the peers. Every timeout in the
preceding hour landed inside the single minute muse was enqueueing, and in the
same window slskd also failed to resolve three OTHER users' searches. Each
enqueue blocks about five seconds waiting on a peer message connection, so five
back-to-back attempts is twenty-five seconds of sustained pressure on one
subsystem. The ladder turned one bad moment into five spent candidates and a
dead request, and made slskd worse for everyone else while doing it.

Attempts are now spaced by MUSE_ATTEMPT_BACKOFF. Failures are classified: "File
not shared" is about a file and still advances at once, while a timeout is about
slskd and, after MUSE_SYSTEMIC_LIMIT of them in a row, stops the walk and leaves
the untried candidates on the row. A retry resumes that ladder instead of
re-running a search whose answer has not changed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app.py

@@ -100,6 +100,17 @@ SEARCH_RETRIES = int(os.getenv("MUSE_SEARCH_RETRIES", "3"))
 # Soulseek results trickle in: what you have at t=5s is much worse than t=15s.
 SEARCH_POLLS = int(os.getenv("MUSE_SEARCH_POLLS", "20"))
 
+# Pause between candidate attempts. Without one, a ladder of 8 walks the whole
+# list inside a single bad slskd window -- each enqueue blocks ~5s on a peer
+# connection, so eight attempts is 40 seconds of hammering one subsystem, and
+# it takes other people's searches down with it.
+ATTEMPT_BACKOFF = float(os.getenv("MUSE_ATTEMPT_BACKOFF", "4"))
+
+# Consecutive systemic failures before we stop walking the ladder. A file that
+# is not shared says something about that file; a timeout says only that slskd
+# is having a moment, and burning eight candidates to learn it once is waste.
+SYSTEMIC_LIMIT = int(os.getenv("MUSE_SYSTEMIC_LIMIT", "2"))
+
 # How many Soulseek searches may run at once.
 #
 # This is a hard cap, not a suggestion. Each search takes 10-40s of polling
@@ -911,6 +922,18 @@ def exhausted_detail(tried: list[dict], attempts: int) -> str:
     return f"tried {n_peers} peer{plural}: {joined}"
 
 
+# Substrings in a failure that describe slskd's state rather than the file's.
+# "File not shared" is about one file and the next candidate is unaffected;
+# a timeout means the client could not talk to anyone, so the next candidate
+# would fail for the same reason and only costs us the candidate.
+SYSTEMIC = ("timed out", "timeout", "connection", "refused", "unreachable")
+
+
+def is_systemic(reason: str) -> bool:
+    r = reason.lower()
+    return any(w in r for w in SYSTEMIC)
+
+
 async def try_next_candidate(job_id: str, artist: str, title: str,
                              album: str | None = None) -> bool:
     """Enqueue the next-best candidate. False once the ladder is exhausted.
@@ -930,7 +953,15 @@ async def try_next_candidate(job_id: str, artist: str, title: str,
     tried = json.loads(row["tried"] or "[]")
     attempts = row["attempts"] or 0
 
+    systemic_run = 0
+    first = True
     while cands and attempts < MAX_ATTEMPTS:
+        # Space the attempts out. The first one is immediate; the rest wait,
+        # because back-to-back enqueues are what saturated slskd.
+        if not first:
+            await asyncio.sleep(ATTEMPT_BACKOFF)
+        first = False
+
         cand = cands.pop(0)
         attempts += 1
         try:
@@ -941,11 +972,33 @@ async def try_next_candidate(job_id: str, artist: str, title: str,
                            "size": cand["size"]}])
                 er.raise_for_status()
         except Exception as e:
+            reason = f"could not reach peer: {errstr(e)}"
             tried.append({"username": cand["username"],
                           "filename": cand["filename"],
-                          "reason": f"could not reach peer: {errstr(e)}"})
+                          "reason": reason})
+            if is_systemic(reason):
+                systemic_run += 1
+                if systemic_run >= SYSTEMIC_LIMIT:
+                    # Stop, and leave the untried candidates on the row so a
+                    # retry resumes the ladder instead of re-running a search
+                    # that already told us the right answer.
+                    with db() as c:
+                        c.execute("UPDATE jobs SET candidates=?, tried=?,"
+                                  " attempts=? WHERE id=?",
+                                  (json.dumps(cands), json.dumps(tried),
+                                   attempts, job_id))
+                    set_state(job_id, "failed",
+                              f"slskd could not reach any peer after "
+                              f"{attempts} attempts -- {len(cands)} candidate"
+                              f"{'' if len(cands) == 1 else 's'} left to try, "
+                              f"so a retry resumes where this stopped")
+                    return False
+            else:
+                systemic_run = 0
             continue
 
+        systemic_run = 0
+
         conf, why = confidence(cand["filename"], artist, title, album)
         with db() as c:
             c.execute(
@@ -1239,8 +1292,20 @@ async def search_worker(n: int):
                 job = c.execute("SELECT * FROM jobs WHERE id=?", (job_id,)).fetchone()
             if job and job["state"] in ("queued", "searching"):
                 set_state(job_id, "searching", "")
-                await pick_and_enqueue(job_id, job["artist"], job["title"],
-                                       job["album"])
+                # A ladder left over from a run that stopped on a slskd stall
+                # is still the right answer -- the search that produced it has
+                # not become wrong. Resume it rather than paying for another.
+                keys = job.keys()
+                left = json.loads(
+                    (job["candidates"] if "candidates" in keys else None) or "[]")
+                if left:
+                    log.info("job %s resuming ladder, %d candidate(s) left",
+                             job_id[:8], len(left))
+                    await try_next_candidate(job_id, job["artist"],
+                                             job["title"], job["album"])
+                else:
+                    await pick_and_enqueue(job_id, job["artist"], job["title"],
+                                           job["album"])
         except Exception as e:
             log.warning("worker %d: %s", n, e)
             try:
@@ -2533,13 +2598,18 @@ async def retry(job_id: str, request: Request):
     if row["state"] != "failed":
         raise HTTPException(409, f"only failed jobs can be retried "
                                  f"(this one is {row['state']})")
-    # Clear the previous attempt, or the ladder resumes mid-way through a
-    # candidate list gathered for a search that has long since expired.
+    # Clear the last transfer, but keep any candidates the previous run did
+    # not get to: those are left behind deliberately when slskd could not
+    # reach anyone, and re-running the search would only rediscover them.
     with db() as c:
+        row = c.execute("SELECT candidates FROM jobs WHERE id=?",
+                        (job_id,)).fetchone()
+        keep = bool(json.loads((row["candidates"] if row else None) or "[]"))
         c.execute("UPDATE jobs SET username=NULL, remote_file=NULL, size=NULL,"
-                  " confidence=NULL, match_why=NULL, candidates=NULL,"
-                  " tried=NULL, attempts=0, started_at=NULL WHERE id=?",
-                  (job_id,))
+                  " confidence=NULL, match_why=NULL, attempts=0,"
+                  " started_at=NULL"
+                  + ("" if keep else ", candidates=NULL, tried=NULL")
+                  + " WHERE id=?", (job_id,))
     set_state(job_id, "queued", "")
     await work_q.put(job_id)
     return {"ok": True}