Age a stalled transfer from created_at when started_at is missing
The stall clock read started_at and returned zero when it was absent, which exempted every row written before that column existed -- precisely the jobs that have been stuck the longest. The live instance had one: a `downloading` job from before the upgrade, its transfer parked in Queued, Remotely, which the new check would have skipped forever. Falls back to created_at, which for a job still in `downloading` is the same question asked a slightly different way. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e1b7b69261
commit
39b0e0bb33
2 files changed
+18
-5
No files matched your search
@@ -1261,8 +1261,16 @@ def transfer_index(payload: list) -> dict[str, dict]:
|
|||||||
|
|
||||||
|
|
||||||
def stalled_for(job: sqlite3.Row) -> float:
|
def stalled_for(job: sqlite3.Row) -> float:
|
||||||
"""Minutes since this transfer last had anything to show for itself."""
|
"""Minutes since this transfer last had anything to show for itself.
|
||||||
started = job["started_at"] if "started_at" in job.keys() else None
|
|
||||||
|
Falls back to created_at, which is what makes this work on rows written
|
||||||
|
before started_at existed. Those are `downloading` with no clock at all,
|
||||||
|
so returning zero for them would exempt precisely the jobs that have been
|
||||||
|
stuck the longest -- the ones this check exists to catch.
|
||||||
|
"""
|
||||||
|
keys = job.keys()
|
||||||
|
started = ((job["started_at"] if "started_at" in keys else None)
|
||||||
|
or (job["created_at"] if "created_at" in keys else None))
|
||||||
if not started:
|
if not started:
|
||||||
return 0.0
|
return 0.0
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -253,10 +253,15 @@ def main() -> int:
|
|||||||
check("a 90-minute-old transfer reads as stalled",
|
check("a 90-minute-old transfer reads as stalled",
|
||||||
app.stalled_for(job_row("j4")) > app.STALL_MINUTES,
|
app.stalled_for(job_row("j4")) > app.STALL_MINUTES,
|
||||||
app.stalled_for(job_row("j4")))
|
app.stalled_for(job_row("j4")))
|
||||||
|
# A row written before started_at existed still has to age out, or the
|
||||||
|
# jobs that have been stuck longest are the only ones exempt from the check.
|
||||||
with app.db() as c:
|
with app.db() as c:
|
||||||
c.execute("UPDATE jobs SET started_at=NULL WHERE id=?", ("j4",))
|
c.execute("UPDATE jobs SET started_at=NULL, created_at=? WHERE id=?",
|
||||||
check("a job with no clock is not spuriously stalled",
|
((datetime.now(timezone.utc) - timedelta(days=2))
|
||||||
app.stalled_for(job_row("j4")) == 0.0)
|
.isoformat(sep=" ")[:19], "j4"))
|
||||||
|
check("a pre-upgrade row falls back to created_at",
|
||||||
|
app.stalled_for(job_row("j4")) > app.STALL_MINUTES,
|
||||||
|
app.stalled_for(job_row("j4")))
|
||||||
|
|
||||||
print(f"\n{ok} passed, {fail} failed")
|
print(f"\n{ok} passed, {fail} failed")
|
||||||
return 1 if fail else 0
|
return 1 if fail else 0
|
||||||
|
|||||||
Reference in new issue
Block a user