Make the container actually deployable, publish to GHCR
The Docker setup built but never came up. Three separate blockers, each found by running the full compose stack against a real Postgres: - The runner stage hand-copied prisma/@prisma/dotenv, which misses the CLI's transitive deps (valibot) and crash-looped the entrypoint. A dedicated migrator stage now installs the CLI at the exact lockfile version with its real dependency tree. - Docker sets HOSTNAME to the container id and Next's standalone server binds to it verbatim, so the app was unreachable on localhost inside the container and the healthcheck could never pass. Pinned to 0.0.0.0. - openssl was missing from the build stages but present at runtime, so Prisma detected a different platform and tried to download a matching query engine at boot, as an unprivileged user, into a root-owned directory. Aligned openssl across every stage that touches Prisma. Verified from a clean volume: migrations apply, both containers report healthy, /api/health returns 200, and the app serves from the image. Also: - Publish multi-arch images to GHCR on push to main and on v* tags, with an SBOM and a signed provenance attestation. - Default compose to the published image; add a build override for working on the app itself. Host port and image are configurable. - Drop two unused dependencies: canvas (19MB of native node-canvas; the pipeline uses @napi-rs/canvas) and multer (routes use formData()). - Add a root .gitignore. There wasn't one, so a real .env was one `git add .` away from being committed. Untrack .DS_Store. - Rewrite the README around what the tool does and how to run it. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Aryan SinghandClaude Opus 5 committed
1 parent
4130ce2e09
commit
7e7e4d9b1c
36 files changed
+2260
-1270
No files matched your search
@@ -2,14 +2,43 @@ import { NextRequest, NextResponse } from "next/server";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { authOptions } from "@/app/api/auth/[...nextauth]/route";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { readFile, writeFile, mkdir } from "fs/promises";
|
||||
import path from "path";
|
||||
import { readFile } from "fs/promises";
|
||||
import {
|
||||
loadPdf,
|
||||
renderPage,
|
||||
sanitizePage,
|
||||
ocrPage,
|
||||
createOcrWorker,
|
||||
type OcrWorker,
|
||||
} from "@/lib/pipeline";
|
||||
import { sourcePdfPath } from "@/lib/storage";
|
||||
import { rateLimit } from "@/lib/rateLimit";
|
||||
import { safeErrorMessage } from "@/lib/safeError";
|
||||
|
||||
/**
|
||||
* Tesseract OCR: runs OCR on EVERY page of the PDF.
|
||||
* Streams progress back to the client via newline-delimited JSON.
|
||||
* Only allowed ONCE per document. Must be explicitly triggered.
|
||||
* Tesseract OCR over every page of a document.
|
||||
*
|
||||
* - OCR runs on the SANITIZED image (the same pipeline the Pixel panel uses),
|
||||
* not the raw render, so hidden low-contrast text is picked up.
|
||||
* - Word-level boxes + per-page confidence are persisted to the pdfPage table.
|
||||
* - Runs are idempotent: an existing run's page rows are cleared and overwritten
|
||||
* rather than rejected.
|
||||
* - Each page is isolated (one bad page does not abort the run) and retried once
|
||||
* with backoff. Progress + per-page results/errors stream as NDJSON.
|
||||
*/
|
||||
|
||||
const OCR_RATE_MAX = 3;
|
||||
const OCR_RATE_WINDOW_MS = 10 * 60 * 1000; // 3 runs / 10 minutes / user
|
||||
|
||||
async function withRetry<T>(fn: () => Promise<T>): Promise<T> {
|
||||
try {
|
||||
return await fn();
|
||||
} catch {
|
||||
await new Promise((r) => setTimeout(r, 600));
|
||||
return fn();
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
@@ -26,25 +55,6 @@ export async function POST(
|
||||
return NextResponse.json({ error: "Document not found" }, { status: 404 });
|
||||
}
|
||||
|
||||
if (doc.ocrRan) {
|
||||
// Return existing results
|
||||
if (doc.ocrResultDir && doc.ocrPageCount > 0) {
|
||||
const pages = Array.from({ length: doc.ocrPageCount }, (_, i) => ({
|
||||
page: i + 1,
|
||||
path: `${doc.ocrResultDir}/page_${i + 1}.txt`,
|
||||
}));
|
||||
return NextResponse.json({
|
||||
pages,
|
||||
pageCount: doc.ocrPageCount,
|
||||
alreadyRan: true,
|
||||
});
|
||||
}
|
||||
return NextResponse.json(
|
||||
{ error: "OCR has already been run", alreadyRan: true },
|
||||
{ status: 409 }
|
||||
);
|
||||
}
|
||||
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { email: session.user.email },
|
||||
});
|
||||
@@ -52,153 +62,116 @@ export async function POST(
|
||||
return NextResponse.json({ error: "Forbidden" }, { status: 403 });
|
||||
}
|
||||
|
||||
try {
|
||||
const pdfPath = path.join(process.cwd(), "public", doc.filePath);
|
||||
const pdfBuffer = await readFile(pdfPath);
|
||||
|
||||
const pdfjsLib = await import("pdfjs-dist/legacy/build/pdf.mjs");
|
||||
const loadingTask = pdfjsLib.getDocument({
|
||||
data: new Uint8Array(pdfBuffer),
|
||||
});
|
||||
const pdfDoc = await loadingTask.promise;
|
||||
const numPages = pdfDoc.numPages;
|
||||
|
||||
const resultsDir = path.join(
|
||||
process.cwd(),
|
||||
"public",
|
||||
"results",
|
||||
user.id,
|
||||
id,
|
||||
"ocr"
|
||||
);
|
||||
await mkdir(resultsDir, { recursive: true });
|
||||
|
||||
const publicDir = `/results/${user.id}/${id}/ocr`;
|
||||
|
||||
// Stream progress to the client via newline-delimited JSON
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
try {
|
||||
const { createCanvas } = await import("@napi-rs/canvas");
|
||||
const Tesseract = await import("tesseract.js");
|
||||
|
||||
const allPages: { page: number; path: string }[] = [];
|
||||
|
||||
for (let pageNum = 1; pageNum <= numPages; pageNum++) {
|
||||
// Send progress event
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
JSON.stringify({
|
||||
type: "progress",
|
||||
page: pageNum,
|
||||
totalPages: numPages,
|
||||
}) + "\n"
|
||||
)
|
||||
);
|
||||
|
||||
const page = await pdfDoc.getPage(pageNum);
|
||||
const scale = 2.0;
|
||||
const viewport = page.getViewport({ scale });
|
||||
const width = Math.floor(viewport.width);
|
||||
const height = Math.floor(viewport.height);
|
||||
|
||||
const canvas = createCanvas(width, height);
|
||||
const ctx = canvas.getContext("2d");
|
||||
|
||||
// Fill white background for accurate OCR
|
||||
ctx.fillStyle = "#ffffff";
|
||||
ctx.fillRect(0, 0, width, height);
|
||||
|
||||
await page.render({
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
canvasContext: ctx as any,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
canvas: canvas as any,
|
||||
viewport,
|
||||
}).promise;
|
||||
|
||||
const imageBuffer = canvas.toBuffer("image/png");
|
||||
const tempImgPath = path.join(
|
||||
resultsDir,
|
||||
`page_${pageNum}_temp.png`
|
||||
);
|
||||
await writeFile(tempImgPath, imageBuffer);
|
||||
|
||||
const { data } = await Tesseract.recognize(tempImgPath, "eng");
|
||||
|
||||
// Save per-page OCR text
|
||||
const ocrFileName = `page_${pageNum}.txt`;
|
||||
const ocrPath = path.join(resultsDir, ocrFileName);
|
||||
await writeFile(ocrPath, data.text, "utf-8");
|
||||
|
||||
const publicPath = `${publicDir}/${ocrFileName}`;
|
||||
allPages.push({ page: pageNum, path: publicPath });
|
||||
|
||||
// Send page-complete event
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
JSON.stringify({
|
||||
type: "page_done",
|
||||
page: pageNum,
|
||||
totalPages: numPages,
|
||||
text: data.text,
|
||||
confidence: data.confidence,
|
||||
path: publicPath,
|
||||
}) + "\n"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Update DB
|
||||
await prisma.pdfDocument.update({
|
||||
where: { id },
|
||||
data: {
|
||||
ocrRan: true,
|
||||
ocrResultDir: publicDir,
|
||||
ocrPageCount: numPages,
|
||||
pageCount: numPages,
|
||||
},
|
||||
});
|
||||
|
||||
// Send final complete event
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
JSON.stringify({
|
||||
type: "complete",
|
||||
pages: allPages,
|
||||
pageCount: numPages,
|
||||
}) + "\n"
|
||||
)
|
||||
);
|
||||
controller.close();
|
||||
} catch (err) {
|
||||
console.error("OCR streaming error:", err);
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
JSON.stringify({
|
||||
type: "error",
|
||||
error: "OCR processing failed",
|
||||
}) + "\n"
|
||||
)
|
||||
);
|
||||
controller.close();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
"Content-Type": "text/plain; charset=utf-8",
|
||||
"Transfer-Encoding": "chunked",
|
||||
"Cache-Control": "no-cache",
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
console.error("OCR error:", e);
|
||||
// Per-user rate limit on OCR runs.
|
||||
const limit = rateLimit(`ocr:${user.id}`, OCR_RATE_MAX, OCR_RATE_WINDOW_MS);
|
||||
if (!limit.allowed) {
|
||||
return NextResponse.json(
|
||||
{ error: "OCR processing failed" },
|
||||
{ status: 500 }
|
||||
{
|
||||
error: `Rate limit reached. Try again in ${limit.retryAfterSeconds}s.`,
|
||||
retryAfterSeconds: limit.retryAfterSeconds,
|
||||
},
|
||||
{ status: 429, headers: { "Retry-After": String(limit.retryAfterSeconds) } }
|
||||
);
|
||||
}
|
||||
|
||||
let pdfDoc: Awaited<ReturnType<typeof loadPdf>>;
|
||||
let numPages: number;
|
||||
try {
|
||||
const pdfBuffer = await readFile(sourcePdfPath(doc.filePath));
|
||||
pdfDoc = await loadPdf(pdfBuffer);
|
||||
numPages = pdfDoc.numPages;
|
||||
} catch (e) {
|
||||
console.error("OCR load error:", e);
|
||||
return NextResponse.json({ error: safeErrorMessage(e) }, { status: 500 });
|
||||
}
|
||||
|
||||
// Idempotent re-run: drop any previously persisted pages for this document.
|
||||
await prisma.pdfPage.deleteMany({ where: { documentId: id } });
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream({
|
||||
async start(controller) {
|
||||
const emit = (obj: unknown) =>
|
||||
controller.enqueue(encoder.encode(JSON.stringify(obj) + "\n"));
|
||||
|
||||
let worker: OcrWorker | null = null;
|
||||
let succeeded = 0;
|
||||
try {
|
||||
worker = await createOcrWorker();
|
||||
|
||||
for (let pageNum = 1; pageNum <= numPages; pageNum++) {
|
||||
emit({ type: "progress", page: pageNum, totalPages: numPages });
|
||||
|
||||
try {
|
||||
const result = await withRetry(async () => {
|
||||
const { buffer } = await renderPage(pdfDoc, pageNum);
|
||||
const sanitized = await sanitizePage(buffer);
|
||||
return ocrPage(worker, sanitized);
|
||||
});
|
||||
|
||||
await prisma.pdfPage.create({
|
||||
data: {
|
||||
documentId: id,
|
||||
pageNumber: pageNum,
|
||||
confidence: result.confidence,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
wordsJson: result.words as any,
|
||||
},
|
||||
});
|
||||
succeeded += 1;
|
||||
|
||||
emit({
|
||||
type: "page_done",
|
||||
page: pageNum,
|
||||
totalPages: numPages,
|
||||
text: result.text,
|
||||
confidence: result.confidence,
|
||||
wordCount: result.words.length,
|
||||
});
|
||||
} catch (pageErr) {
|
||||
// One bad page must not kill the whole run.
|
||||
console.error(`OCR page ${pageNum} failed:`, pageErr);
|
||||
emit({
|
||||
type: "page_error",
|
||||
page: pageNum,
|
||||
totalPages: numPages,
|
||||
error: safeErrorMessage(pageErr),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await prisma.pdfDocument.update({
|
||||
where: { id },
|
||||
data: {
|
||||
ocrRan: true,
|
||||
ocrResultDir: null,
|
||||
ocrPageCount: succeeded,
|
||||
pageCount: numPages,
|
||||
},
|
||||
});
|
||||
|
||||
emit({ type: "complete", pageCount: numPages, succeeded });
|
||||
} catch (err) {
|
||||
console.error("OCR streaming error:", err);
|
||||
emit({ type: "error", error: safeErrorMessage(err) });
|
||||
} finally {
|
||||
if (worker) {
|
||||
try {
|
||||
await worker.terminate();
|
||||
} catch {
|
||||
/* ignore terminate errors */
|
||||
}
|
||||
}
|
||||
controller.close();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
"Content-Type": "text/plain; charset=utf-8",
|
||||
"Transfer-Encoding": "chunked",
|
||||
"Cache-Control": "no-cache",
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { getServerSession } from "next-auth";
|
||||
import { authOptions } from "@/app/api/auth/[...nextauth]/route";
|
||||
import { prisma } from "@/lib/prisma";
|
||||
import { wordsToText, type OcrWord } from "@/lib/pipeline";
|
||||
|
||||
/**
|
||||
* Persisted OCR words for a single page (used by the confidence-heatmap
|
||||
* overlay so it survives reloads). Returns per-word boxes + confidence, the
|
||||
* page-level confidence, and text reconstructed from the words.
|
||||
*
|
||||
* GET /api/documents/[id]/ocr/words?page=N
|
||||
*/
|
||||
export async function GET(
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const session = await getServerSession(authOptions);
|
||||
if (!session?.user?.email) {
|
||||
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
const page = Number(req.nextUrl.searchParams.get("page"));
|
||||
if (!Number.isInteger(page) || page < 1) {
|
||||
return NextResponse.json({ error: "Invalid page" }, { status: 400 });
|
||||
}
|
||||
|
||||
const doc = await prisma.pdfDocument.findUnique({ where: { id } });
|
||||
if (!doc) {
|
||||
return NextResponse.json({ error: "Document not found" }, { status: 404 });
|
||||
}
|
||||
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { email: session.user.email },
|
||||
});
|
||||
if (!user || doc.userId !== user.id) {
|
||||
return NextResponse.json({ error: "Forbidden" }, { status: 403 });
|
||||
}
|
||||
|
||||
const row = await prisma.pdfPage.findUnique({
|
||||
where: { documentId_pageNumber: { documentId: id, pageNumber: page } },
|
||||
});
|
||||
if (!row) {
|
||||
return NextResponse.json(
|
||||
{ error: "No OCR data for this page" },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
|
||||
const words = (row.wordsJson as unknown as OcrWord[]) ?? [];
|
||||
return NextResponse.json({
|
||||
page,
|
||||
confidence: row.confidence,
|
||||
words,
|
||||
text: wordsToText(words),
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user