Freeway: self-hosted OpenAI-compatible gateway for free-tier providers
One endpoint in front of every free LLM tier, with automatic failover and no provider key ever leaving the box. Providers are data, not code: adding one is a JSON file plus an env var, never a TypeScript change. Ships 12 providers pre-written; `verify-provider --apply` derives the live model list from your own key. - core: registry, bucketed quota meter (rolling + UTC-calendar windows), key-pool store, 6-step model router with diagnostic rejection reasons - gateway: node:http, failover that commits to a stream only after 200 headers, SSE passthrough with usage sniffing, virtual keys with admin/inference roles - context engine: sqlite sessions and a 5-rung refit ladder, so a 128k thread keeps working when it fails over to an 8k model - cache: exact (sha256, safe-by-default) + opt-in semantic (flat Float32Array) - guard: inbound limits, quota forecasting, limit calibration from upstream headers, key health, secret/PII/injection scanning - dashboard: one self-contained HTML file, live SSE request log, token usage - sdk: zero-dep typed client, drop-in for the OpenAI SDK Zero runtime dependencies. 326 tests, 27-check smoke suite against a scripted mock upstream so routing is testable without spending quota. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Aryan SinghandClaude Opus 5 committed
commit
cb985dcf9c
86 files changed
+16350
No files matched your search
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "@freeway/gateway",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "./dist/server.js",
|
||||
"types": "./dist/server.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/server.d.ts",
|
||||
"default": "./dist/server.js"
|
||||
}
|
||||
},
|
||||
"files": ["dist", "public"],
|
||||
"dependencies": {
|
||||
"@freeway/core": "*"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "node --test test/*.test.ts"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,991 @@
|
||||
<!doctype html>
|
||||
<html lang="en" data-theme="dark">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Freeway</title>
|
||||
<style>
|
||||
/* An ops panel: dense, monospaced numbers, nothing that moves without reason.
|
||||
No framework, no CDN, no build step — this file is the whole dashboard. */
|
||||
:root {
|
||||
--bg:#0a0c0f; --panel:#12151a; --panel-2:#171b21; --line:#242a33; --line-2:#2f3742;
|
||||
--fg:#dbe2ea; --fg-dim:#98a3b1; --fg-faint:#69737f;
|
||||
--green:#3fb950; --amber:#d29922; --red:#f85149; --blue:#58a6ff; --purple:#bc8cff;
|
||||
--mono:ui-monospace,SFMono-Regular,"SF Mono",Menlo,Consolas,monospace;
|
||||
--sans:system-ui,-apple-system,"Segoe UI",Roboto,sans-serif;
|
||||
}
|
||||
*{box-sizing:border-box}
|
||||
html,body{margin:0;padding:0}
|
||||
body{
|
||||
background:var(--bg); color:var(--fg); font:13px/1.45 var(--sans);
|
||||
font-variant-numeric:tabular-nums; -webkit-font-smoothing:antialiased;
|
||||
}
|
||||
.num,code,pre,.mono{font-family:var(--mono); font-variant-numeric:tabular-nums}
|
||||
a{color:var(--blue); text-decoration:none}
|
||||
a:hover{text-decoration:underline}
|
||||
button{font:inherit; cursor:pointer}
|
||||
|
||||
/* ---- top bar ---- */
|
||||
header{
|
||||
position:sticky; top:0; z-index:20; background:rgba(10,12,15,.92);
|
||||
backdrop-filter:blur(8px); border-bottom:1px solid var(--line);
|
||||
display:flex; align-items:center; gap:18px; padding:10px 16px; flex-wrap:wrap;
|
||||
}
|
||||
.brand{font-weight:700; letter-spacing:-.02em; font-size:15px; display:flex; align-items:center; gap:8px}
|
||||
.brand svg{display:block}
|
||||
.stat{display:flex; flex-direction:column; line-height:1.2}
|
||||
.stat b{font-family:var(--mono); font-size:14px; font-weight:600}
|
||||
.stat span{font-size:10px; text-transform:uppercase; letter-spacing:.07em; color:var(--fg-faint)}
|
||||
.spacer{flex:1}
|
||||
.btn{
|
||||
background:var(--panel-2); border:1px solid var(--line-2); color:var(--fg);
|
||||
padding:5px 11px; border-radius:6px; font-size:12px;
|
||||
}
|
||||
.btn:hover{background:#1e232b; border-color:#3a434f}
|
||||
.btn.on{border-color:var(--green); color:var(--green)}
|
||||
.live{display:flex; align-items:center; gap:6px; font-size:11px; color:var(--fg-faint)}
|
||||
.pulse{width:7px;height:7px;border-radius:50%;background:var(--green)}
|
||||
.pulse.stale{background:var(--amber)}
|
||||
.pulse.dead{background:var(--red)}
|
||||
|
||||
main{padding:16px; max-width:1800px; margin:0 auto}
|
||||
section{margin-bottom:26px}
|
||||
h2{
|
||||
font-size:11px; text-transform:uppercase; letter-spacing:.09em; color:var(--fg-faint);
|
||||
margin:0 0 10px; display:flex; align-items:center; gap:10px; font-weight:600;
|
||||
}
|
||||
h2::after{content:""; flex:1; height:1px; background:var(--line)}
|
||||
|
||||
/* ---- banners ---- */
|
||||
.banner{
|
||||
border:1px solid var(--line-2); border-left-width:3px; border-radius:6px;
|
||||
padding:9px 12px; margin-bottom:10px; font-size:12px;
|
||||
}
|
||||
.banner.err{border-left-color:var(--red); background:rgba(248,81,73,.06)}
|
||||
.banner.warn{border-left-color:var(--amber); background:rgba(210,153,34,.06)}
|
||||
.banner.info{border-left-color:var(--blue); background:rgba(88,166,255,.06)}
|
||||
.banner b{font-family:var(--mono)}
|
||||
|
||||
/* ---- provider cards ---- */
|
||||
.cards{display:grid; grid-template-columns:repeat(auto-fill,minmax(400px,1fr)); gap:12px}
|
||||
.card{
|
||||
background:var(--panel); border:1px solid var(--line); border-radius:9px;
|
||||
padding:13px 14px; display:flex; flex-direction:column; gap:11px;
|
||||
}
|
||||
.card.off{opacity:.55}
|
||||
.card.unconfigured{border-style:dashed}
|
||||
.card-head{display:flex; align-items:flex-start; gap:9px}
|
||||
.dot{width:9px;height:9px;border-radius:50%;flex:none;margin-top:5px}
|
||||
.dot.green{background:var(--green); box-shadow:0 0 7px rgba(63,185,80,.55)}
|
||||
.dot.amber{background:var(--amber); box-shadow:0 0 7px rgba(210,153,34,.5)}
|
||||
.dot.red{background:var(--red); box-shadow:0 0 7px rgba(248,81,73,.55)}
|
||||
.dot.grey{background:#454d58}
|
||||
.card-title{flex:1; min-width:0}
|
||||
.card-title h3{margin:0; font-size:14px; font-weight:600; letter-spacing:-.01em}
|
||||
.card-title .sub{font-family:var(--mono); font-size:10.5px; color:var(--fg-faint); word-break:break-all; margin-top:2px}
|
||||
.toggle{
|
||||
position:relative; width:34px; height:19px; border-radius:19px; border:1px solid var(--line-2);
|
||||
background:#1c212a; flex:none; padding:0; transition:background .12s,border-color .12s;
|
||||
}
|
||||
.toggle::after{
|
||||
content:""; position:absolute; top:2px; left:2px; width:13px; height:13px; border-radius:50%;
|
||||
background:#6b7683; transition:transform .12s, background .12s;
|
||||
}
|
||||
.toggle[aria-pressed="true"]{background:rgba(63,185,80,.22); border-color:var(--green)}
|
||||
.toggle[aria-pressed="true"]::after{transform:translateX(15px); background:var(--green)}
|
||||
|
||||
.metrics{display:flex; gap:16px; flex-wrap:wrap; font-size:11px}
|
||||
.metrics div{display:flex; flex-direction:column; line-height:1.25}
|
||||
.metrics b{font-family:var(--mono); font-size:13px; font-weight:600}
|
||||
.metrics span{color:var(--fg-faint); font-size:9.5px; text-transform:uppercase; letter-spacing:.06em}
|
||||
.ok-c{color:var(--green)} .fail-c{color:var(--red)} .dim{color:var(--fg-dim)}
|
||||
|
||||
/* ---- quota bars ---- */
|
||||
.bars{display:flex; flex-direction:column; gap:8px}
|
||||
.bar{
|
||||
display:grid; grid-template-columns:56px 1fr auto; grid-template-rows:auto auto;
|
||||
gap:3px 9px; align-items:center; font-size:10.5px;
|
||||
}
|
||||
.bar-key{grid-area:1/1; font-family:var(--mono); color:var(--fg-dim); text-transform:uppercase; letter-spacing:.02em; overflow:hidden; text-overflow:ellipsis}
|
||||
/* display:block matters — a percentage width on an inline element is ignored,
|
||||
which silently renders every bar at zero. */
|
||||
.bar-track{grid-area:1/2; display:block; height:6px; background:#212832; border-radius:3px; overflow:hidden}
|
||||
.bar-fill{display:block; height:100%; border-radius:3px; min-width:2px; transition:width .3s ease}
|
||||
.bar-fill.zero{min-width:0}
|
||||
.bar-fill.lo{background:var(--green)} .bar-fill.mid{background:var(--amber)} .bar-fill.hi{background:var(--red)}
|
||||
.bar-meta{grid-area:1/3; font-family:var(--mono); color:var(--fg-dim); white-space:nowrap; font-size:10px}
|
||||
.bar-window{grid-area:2/2; color:var(--fg-faint); font-size:9.5px}
|
||||
|
||||
/* ---- keys ---- */
|
||||
.keys{display:flex; flex-direction:column; gap:4px}
|
||||
.key-row{
|
||||
display:flex; align-items:center; gap:8px; font-family:var(--mono); font-size:11px;
|
||||
padding:4px 7px; background:var(--panel-2); border-radius:5px; border:1px solid transparent;
|
||||
}
|
||||
.key-row.bad{border-color:rgba(248,81,73,.35)}
|
||||
.key-row.cool{border-color:rgba(210,153,34,.35)}
|
||||
.pill{
|
||||
font-size:9px; padding:1px 5px; border-radius:3px; text-transform:uppercase;
|
||||
letter-spacing:.05em; font-family:var(--sans); font-weight:600;
|
||||
}
|
||||
.pill.ok{background:rgba(63,185,80,.16); color:var(--green)}
|
||||
.pill.invalid{background:rgba(248,81,73,.16); color:var(--red)}
|
||||
.pill.rate-limited,.pill.cooling{background:rgba(210,153,34,.16); color:var(--amber)}
|
||||
.pill.unknown{background:#242a33; color:var(--fg-faint)}
|
||||
.pill.grey{background:#242a33; color:var(--fg-dim)}
|
||||
.pill.blue{background:rgba(88,166,255,.14); color:var(--blue)}
|
||||
.pill.purple{background:rgba(188,140,255,.14); color:var(--purple)}
|
||||
|
||||
.err-line{
|
||||
font-family:var(--mono); font-size:10.5px; color:#ff9d96;
|
||||
background:rgba(248,81,73,.07); border-radius:5px; padding:6px 8px; word-break:break-word;
|
||||
}
|
||||
.advice{
|
||||
font-size:11.5px; background:rgba(210,153,34,.07); border-radius:6px; padding:8px 10px;
|
||||
display:flex; flex-direction:column; gap:6px;
|
||||
}
|
||||
.advice code{
|
||||
background:#0a0c0f; border:1px solid var(--line-2); border-radius:4px;
|
||||
padding:3px 6px; font-size:10.5px; display:inline-block;
|
||||
}
|
||||
.copy{background:none;border:1px solid var(--line-2);color:var(--fg-dim);border-radius:4px;padding:2px 6px;font-size:10px}
|
||||
.copy:hover{color:var(--fg); border-color:#3a434f}
|
||||
.links{display:flex; gap:10px; font-size:11px; margin-top:auto; padding-top:2px}
|
||||
.paste{display:flex; gap:6px}
|
||||
.paste input{flex:1; min-width:0; font-size:11px; padding:5px 8px}
|
||||
.probe{
|
||||
display:flex; flex-direction:column; gap:3px; background:var(--panel-2);
|
||||
border-radius:6px; padding:8px 9px; max-height:240px; overflow:auto;
|
||||
}
|
||||
.probe-head{display:flex; gap:8px; align-items:center; font-size:11px; margin-bottom:3px}
|
||||
.probe-row{display:flex; gap:6px; align-items:center; font-family:var(--mono); font-size:10.5px; padding:2px 0}
|
||||
.usage{display:grid; grid-template-columns:repeat(auto-fit,minmax(210px,1fr)); gap:12px; margin-bottom:12px}
|
||||
.ucard{background:var(--panel); border:1px solid var(--line); border-radius:9px; padding:12px 14px}
|
||||
.ucard .big{font-family:var(--mono); font-size:24px; font-weight:600; letter-spacing:-.02em; line-height:1.1}
|
||||
.ucard .lbl{font-size:9.5px; text-transform:uppercase; letter-spacing:.08em; color:var(--fg-faint); margin-top:3px}
|
||||
.ucard .sub{font-size:10.5px; color:var(--fg-dim); margin-top:5px; font-family:var(--mono)}
|
||||
.split{display:grid; grid-template-columns:1fr 1fr; gap:12px}
|
||||
@media (max-width:900px){.split{grid-template-columns:1fr}}
|
||||
.bd{background:var(--panel); border:1px solid var(--line); border-radius:9px; overflow:hidden}
|
||||
.bd h4{margin:0; padding:8px 12px; font-size:9.5px; text-transform:uppercase; letter-spacing:.08em;
|
||||
color:var(--fg-faint); border-bottom:1px solid var(--line); background:var(--panel-2)}
|
||||
.brow{display:grid; grid-template-columns:1fr auto; gap:10px; padding:6px 12px; font-size:11.5px; align-items:center}
|
||||
.brow + .brow{border-top:1px solid #1a1e25}
|
||||
.brow .n{font-family:var(--mono); color:var(--fg-dim); white-space:nowrap; font-size:11px}
|
||||
.msg{
|
||||
background:#0d1015; border-radius:6px; padding:9px 11px; margin:6px 0;
|
||||
font-size:11.5px; line-height:1.5; white-space:pre-wrap; word-break:break-word;
|
||||
max-height:280px; overflow:auto; border-left:2px solid var(--line-2);
|
||||
}
|
||||
.msg.out{border-left-color:var(--green)}
|
||||
.msg-h{font-size:9.5px; text-transform:uppercase; letter-spacing:.07em; color:var(--fg-faint); margin-top:8px}
|
||||
.alert{
|
||||
display:flex; gap:9px; align-items:center; padding:7px 11px; border-radius:6px;
|
||||
border:1px solid var(--line); border-left-width:3px; margin-bottom:6px; font-size:12px;
|
||||
background:var(--panel);
|
||||
}
|
||||
.alert.exhausted,.alert.critical{border-left-color:var(--red); background:rgba(248,81,73,.06)}
|
||||
.alert.warn{border-left-color:var(--amber); background:rgba(210,153,34,.06)}
|
||||
.alert .src{font-size:9px; text-transform:uppercase; letter-spacing:.06em; color:var(--fg-faint); font-weight:600}
|
||||
.forecast{grid-area:2/3; font-size:9.5px; color:var(--fg-faint); text-align:right}
|
||||
|
||||
/* ---- tables ---- */
|
||||
.toolbar{display:flex; gap:8px; align-items:center; margin-bottom:8px; flex-wrap:wrap}
|
||||
input[type=search],input[type=password]{
|
||||
background:var(--panel); border:1px solid var(--line-2); color:var(--fg);
|
||||
border-radius:6px; padding:6px 10px; font:12px var(--mono); min-width:240px;
|
||||
}
|
||||
input:focus{outline:none; border-color:var(--blue)}
|
||||
.tbl-wrap{border:1px solid var(--line); border-radius:8px; overflow:auto; max-height:520px; background:var(--panel)}
|
||||
table{width:100%; border-collapse:collapse; font-size:11.5px}
|
||||
thead th{
|
||||
position:sticky; top:0; background:var(--panel-2); text-align:left; font-weight:600;
|
||||
font-size:9.5px; text-transform:uppercase; letter-spacing:.07em; color:var(--fg-faint);
|
||||
padding:7px 10px; border-bottom:1px solid var(--line); white-space:nowrap; z-index:2;
|
||||
}
|
||||
tbody td{padding:6px 10px; border-bottom:1px solid #1a1e25; vertical-align:top}
|
||||
tbody tr:last-child td{border-bottom:none}
|
||||
tbody tr.clickable{cursor:pointer}
|
||||
tbody tr.clickable:hover{background:var(--panel-2)}
|
||||
td.n{font-family:var(--mono); white-space:nowrap}
|
||||
.caps{display:flex; gap:3px; flex-wrap:wrap}
|
||||
.status-200{color:var(--green)} .status-4xx{color:var(--amber)} .status-5xx{color:var(--red)}
|
||||
.attempts-detail{background:#0d1015; font-family:var(--mono); font-size:10.5px; padding:8px 10px}
|
||||
.attempts-detail div{padding:2px 0; color:var(--fg-dim)}
|
||||
.empty{padding:26px; text-align:center; color:var(--fg-faint); font-size:12px}
|
||||
.gate{max-width:420px; margin:70px auto; background:var(--panel); border:1px solid var(--line);
|
||||
border-radius:10px; padding:22px; display:flex; flex-direction:column; gap:12px}
|
||||
.hide{display:none !important}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<header>
|
||||
<div class="brand">
|
||||
<svg width="17" height="17" viewBox="0 0 16 16" fill="none" aria-hidden="true">
|
||||
<path d="M2 13 L8 3 L14 13" stroke="#58a6ff" stroke-width="1.6" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<path d="M5.2 8.3 H10.8" stroke="#3fb950" stroke-width="1.6" stroke-linecap="round"/>
|
||||
</svg>
|
||||
Freeway
|
||||
</div>
|
||||
<div class="stat"><b id="s-providers">–</b><span>providers</span></div>
|
||||
<div class="stat"><b id="s-models">–</b><span>models</span></div>
|
||||
<div class="stat"><b id="s-ok" class="ok-c">–</b><span>ok</span></div>
|
||||
<div class="stat"><b id="s-fail" class="fail-c">–</b><span>failed</span></div>
|
||||
<div class="stat"><b id="s-strategy">–</b><span>strategy</span></div>
|
||||
<div class="stat"><b id="s-uptime">–</b><span>uptime</span></div>
|
||||
<div class="spacer"></div>
|
||||
<div class="live">
|
||||
<span class="pulse" id="pulse"></span><span id="pulse-text">live</span>
|
||||
<span class="pill grey" id="live-badge" style="transition:transform .18s ease">connecting</span>
|
||||
</div>
|
||||
<button class="btn" id="btn-pause">pause</button>
|
||||
<button class="btn" id="btn-reload">reload providers</button>
|
||||
</header>
|
||||
|
||||
<div id="gate" class="gate hide">
|
||||
<div><b>This gateway requires a virtual key.</b></div>
|
||||
<div style="color:var(--fg-dim);font-size:12px">
|
||||
The dashboard reads <code class="mono">/api/*</code>, which is administrative — it needs
|
||||
<code class="mono">FREEWAY_ADMIN_KEY</code>, not the inference key your apps use.
|
||||
It is kept in this browser's localStorage and never sent anywhere but this gateway.
|
||||
</div>
|
||||
<input type="password" id="gate-key" placeholder="fw-…" autocomplete="off">
|
||||
<button class="btn" id="gate-go">unlock</button>
|
||||
<div id="gate-err" class="err-line hide"></div>
|
||||
</div>
|
||||
|
||||
<main id="app" class="hide">
|
||||
<div id="banners"></div>
|
||||
|
||||
<section>
|
||||
<h2>Token usage <span style="text-transform:none;letter-spacing:0;font-weight:400" id="usage-window"></span></h2>
|
||||
<div class="usage" id="usage-cards"></div>
|
||||
<div class="split">
|
||||
<div class="bd"><h4>By provider</h4><div id="by-provider"></div></div>
|
||||
<div class="bd"><h4>By model</h4><div id="by-model"></div></div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="alerts-section" class="hide">
|
||||
<h2>Alerts</h2>
|
||||
<div id="alerts"></div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Providers</h2>
|
||||
<div class="cards" id="cards"></div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Model catalog</h2>
|
||||
<div class="toolbar">
|
||||
<input type="search" id="model-search" placeholder="filter by model, provider, alias or capability…">
|
||||
<span id="model-count" style="font-size:11px;color:var(--fg-faint)"></span>
|
||||
</div>
|
||||
<div class="tbl-wrap">
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>Provider</th><th>Model</th><th>Aliases</th><th>Capabilities</th><th>Context</th><th>State</th>
|
||||
</tr></thead>
|
||||
<tbody id="models-body"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>Request log</h2>
|
||||
<div class="toolbar">
|
||||
<input type="search" id="log-search" placeholder="filter the log…">
|
||||
<span style="font-size:11px;color:var(--fg-faint)">click any row to see the prompt, the response and its tokens</span>
|
||||
</div>
|
||||
<div class="tbl-wrap">
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>Time</th><th>Requested</th><th>Resolution</th><th>Served by</th>
|
||||
<th>Att</th><th>Status</th><th>Latency</th><th>Tokens</th>
|
||||
</tr></thead>
|
||||
<tbody id="log-body"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
'use strict';
|
||||
|
||||
const POLL_MS = 3000;
|
||||
const KEY_STORAGE = 'freeway.key';
|
||||
|
||||
let apiKey = localStorage.getItem(KEY_STORAGE) || '';
|
||||
let paused = false;
|
||||
let timer = null;
|
||||
let lastOk = 0;
|
||||
const expanded = new Set();
|
||||
let state = { providers: [], logs: [], stats: null, issues: [], probes: {}, alerts: null, usage: null };
|
||||
|
||||
/* Provider labels, notes and requested model strings are all attacker-reachable
|
||||
in principle (a model string comes straight off the wire), so nothing is
|
||||
interpolated into HTML without going through here. */
|
||||
function esc(v) {
|
||||
return String(v ?? '').replace(/[&<>"']/g, c => (
|
||||
{ '&':'&', '<':'<', '>':'>', '"':'"', "'":''' }[c]
|
||||
));
|
||||
}
|
||||
|
||||
async function api(path, options) {
|
||||
const opts = Object.assign({ headers: {} }, options);
|
||||
if (apiKey) opts.headers['authorization'] = 'Bearer ' + apiKey;
|
||||
if (opts.body) opts.headers['content-type'] = 'application/json';
|
||||
const res = await fetch(path, opts);
|
||||
// 403 means the key was valid but is inference-only; the dashboard needs the
|
||||
// admin key, so both statuses land on the same prompt.
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
const e = new Error(res.status === 403 ? 'this endpoint needs the admin key' : 'unauthorized');
|
||||
e.unauthorized = true;
|
||||
throw e;
|
||||
}
|
||||
if (!res.ok) throw new Error(path + ' → ' + res.status);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- formatting
|
||||
|
||||
function fmtNum(n) {
|
||||
if (n === null || n === undefined) return '–';
|
||||
if (n >= 1e9) return (n / 1e9).toFixed(1) + 'B';
|
||||
if (n >= 1e6) return (n / 1e6).toFixed(1) + 'M';
|
||||
if (n >= 1e3) return (n / 1e3).toFixed(1) + 'K';
|
||||
return String(Math.round(n));
|
||||
}
|
||||
function fmtMs(ms) {
|
||||
if (ms === null || ms === undefined) return '–';
|
||||
if (ms < 1000) return Math.round(ms) + 'ms';
|
||||
return (ms / 1000).toFixed(1) + 's';
|
||||
}
|
||||
function fmtDuration(sec) {
|
||||
if (sec < 60) return sec + 's';
|
||||
if (sec < 3600) return Math.floor(sec / 60) + 'm';
|
||||
if (sec < 86400) return Math.floor(sec / 3600) + 'h';
|
||||
return Math.floor(sec / 86400) + 'd';
|
||||
}
|
||||
function fmtTime(ts) {
|
||||
const d = new Date(ts);
|
||||
return d.toTimeString().slice(0, 8) + '.' + String(d.getMilliseconds()).padStart(3, '0');
|
||||
}
|
||||
function humaniseMs(ms) {
|
||||
if (ms < 60000) return Math.max(1, Math.round(ms / 1000)) + 's';
|
||||
if (ms < 3600000) return Math.round(ms / 60000) + 'm';
|
||||
if (ms < 86400000) return (ms / 3600000).toFixed(1) + 'h';
|
||||
return (ms / 86400000).toFixed(1) + 'd';
|
||||
}
|
||||
|
||||
function barClass(pct) { return pct < 60 ? 'lo' : pct <= 85 ? 'mid' : 'hi'; }
|
||||
|
||||
/* `creditsPerDay` does not fit the label column, and "credits" is what it is. */
|
||||
const BAR_LABELS = { creditsPerDay: 'credits' };
|
||||
function barLabel(key) { return BAR_LABELS[key] || key; }
|
||||
|
||||
/* A key's `status` is the last thing that happened to it, which goes stale: a
|
||||
key benched by a 429 still reads "rate-limited" long after its cooldown has
|
||||
run out and it is usable again. Only `invalid` is worth showing past its
|
||||
timer, because a rejected credential really is still suspect. */
|
||||
function keyStatus(k) {
|
||||
if (k.cooldownMsRemaining > 0) return k.status;
|
||||
if (k.status === 'rate-limited' || k.status === 'cooling') return 'ok';
|
||||
return k.status;
|
||||
}
|
||||
|
||||
/* Green only when it can actually serve traffic right now. An unconfigured
|
||||
provider is amber, not red: nothing is broken, it just needs a key. */
|
||||
function health(p) {
|
||||
if (!p.enabled) return { dot: 'grey', label: 'disabled' };
|
||||
if (!p.configured) return { dot: 'amber', label: 'needs a key' };
|
||||
if (p.cooldownMsRemaining > 0) return { dot: 'red', label: 'cooling ' + fmtMs(p.cooldownMsRemaining) };
|
||||
const live = p.keys.filter(k => k.cooldownMsRemaining === 0).length;
|
||||
if (p.keys.length > 0 && live === 0) return { dot: 'red', label: 'all keys benched' };
|
||||
if (live < p.keys.length) return { dot: 'amber', label: live + '/' + p.keys.length + ' keys live' };
|
||||
if (p.fail > 0 && p.fail / Math.max(1, p.ok + p.fail) > 0.2) return { dot: 'amber', label: 'degraded' };
|
||||
return { dot: 'green', label: 'ready' };
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ rendering
|
||||
|
||||
function renderBanners() {
|
||||
const out = [];
|
||||
const errs = state.issues.filter(i => i.level === 'error');
|
||||
const warns = state.issues.filter(i => i.level === 'warning');
|
||||
|
||||
for (const i of errs) {
|
||||
out.push('<div class="banner err"><b>' + esc(i.file) + '</b> was skipped — ' + esc(i.message) + '</div>');
|
||||
}
|
||||
for (const i of warns) {
|
||||
out.push('<div class="banner warn"><b>' + esc(i.file) + '</b> — ' + esc(i.message) + '</div>');
|
||||
}
|
||||
|
||||
const unconfigured = state.providers.filter(p => !p.configured);
|
||||
if (unconfigured.length > 0 && errs.length === 0) {
|
||||
out.push('<div class="banner info">' + unconfigured.length +
|
||||
' provider' + (unconfigured.length === 1 ? '' : 's') +
|
||||
' waiting for credentials — each card below names the variable to set.</div>');
|
||||
}
|
||||
document.getElementById('banners').innerHTML = out.join('');
|
||||
}
|
||||
|
||||
function renderBars(bars, providerId) {
|
||||
if (!bars || bars.length === 0) {
|
||||
return '<div class="bar-window" style="color:var(--fg-faint)">no published limits — usage is unmetered here</div>';
|
||||
}
|
||||
return '<div class="bars">' + bars.map(b => {
|
||||
const pct = Math.min(100, b.pct);
|
||||
const reset = b.resetsAt ? ' · resets in ' + fmtDuration(Math.max(0, Math.round((b.resetsAt - Date.now()) / 1000))) : '';
|
||||
return '<div class="bar">' +
|
||||
'<span class="bar-key" title="' + esc(b.key) + '">' + esc(barLabel(b.key)) + '</span>' +
|
||||
'<span class="bar-track"><span class="bar-fill ' + barClass(pct) + (pct === 0 ? ' zero' : '') +
|
||||
'" style="width:' + pct.toFixed(1) + '%"></span></span>' +
|
||||
'<span class="bar-meta">' + fmtNum(b.used) + ' / ' + fmtNum(b.limit) + ' · ' + pct.toFixed(0) + '%</span>' +
|
||||
'<span class="bar-window">' + esc(b.window) + reset + '</span>' +
|
||||
(() => {
|
||||
const f = forecastFor(providerId, b.key);
|
||||
if (!f || f.exhaustsInMs === null) return '';
|
||||
return '<span class="forecast" title="' + esc(f.message) + '">dry in ' + esc(humaniseMs(f.exhaustsInMs)) + '</span>';
|
||||
})() +
|
||||
'</div>';
|
||||
}).join('') + '</div>';
|
||||
}
|
||||
|
||||
function renderKeys(p) {
|
||||
if (p.authType === 'none') return '<div class="bar-window">no key required</div>';
|
||||
if (p.keys.length === 0) return '';
|
||||
return '<div class="keys">' + p.keys.map(k => {
|
||||
const status = keyStatus(k);
|
||||
const cls = status === 'invalid' ? 'bad' : k.cooldownMsRemaining > 0 ? 'cool' : '';
|
||||
const cooling = k.cooldownMsRemaining > 0 ? '<span class="pill cooling">' + fmtMs(k.cooldownMsRemaining) + '</span>' : '';
|
||||
return '<div class="key-row ' + cls + '" title="' + esc(k.lastError || '') + '">' +
|
||||
'<span>' + esc(k.masked) + '</span>' +
|
||||
'<span class="pill ' + esc(status) + '">' + esc(status) + '</span>' +
|
||||
cooling +
|
||||
'<span style="flex:1"></span>' +
|
||||
'<span class="ok-c">' + k.ok + '</span><span style="color:var(--fg-faint)">/</span>' +
|
||||
'<span class="' + (k.fail ? 'fail-c' : 'dim') + '">' + k.fail + '</span>' +
|
||||
'<span class="dim">' + fmtMs(k.latencyMs) + '</span>' +
|
||||
'<span class="pill grey">' + esc(k.source) + '</span>' +
|
||||
'</div>';
|
||||
}).join('') + '</div>';
|
||||
}
|
||||
|
||||
/* An unconfigured provider gets a full card, not a hidden entry. Finding out a
|
||||
provider is misconfigured should never require reading the server logs. */
|
||||
function renderAdvice(p) {
|
||||
if (p.configured) return '';
|
||||
const envLine = p.envKeys && p.envKeys.length ? p.envKeys[0] + '=' : '';
|
||||
const needsEnvOnly = p.configError && /environment variable/.test(p.configError);
|
||||
|
||||
return '<div class="advice">' +
|
||||
'<div>' + esc(p.configError || 'not configured') + '</div>' +
|
||||
(envLine
|
||||
? '<div style="display:flex;gap:6px;align-items:center;flex-wrap:wrap">' +
|
||||
'<code>' + esc(envLine) + '<your key></code>' +
|
||||
'<button class="copy" data-copy="' + esc(envLine) + '">copy</button></div>'
|
||||
: '') +
|
||||
// A missing ${ENV} in the baseUrl cannot be fixed by pasting a key, so the
|
||||
// paste box is only offered when a key is genuinely the missing piece.
|
||||
(needsEnvOnly || p.authType === 'none' ? '' :
|
||||
'<div class="paste">' +
|
||||
'<input type="password" placeholder="paste a key to add it now…" data-key-input="' + esc(p.id) + '" autocomplete="off" spellcheck="false">' +
|
||||
'<button class="btn" data-key-save="' + esc(p.id) + '">save</button>' +
|
||||
'</div>' +
|
||||
'<div class="bar-window">stored in data/keys.json (0600), masked everywhere, never logged</div>') +
|
||||
'</div>';
|
||||
}
|
||||
|
||||
function renderProbe(p) {
|
||||
const probe = state.probes[p.id];
|
||||
if (!probe) return '';
|
||||
if (probe.loading) return '<div class="probe"><span class="dim">probing ' + esc(p.id) + '…</span></div>';
|
||||
if (probe.error) return '<div class="err-line">probe failed: ' + esc(probe.error) + '</div>';
|
||||
|
||||
return '<div class="probe">' + probe.results.map(r => {
|
||||
const known = p.models.map(m => m.id);
|
||||
const rows = r.models.map(m => {
|
||||
const cls = m.status === 'ok' ? 'ok' : m.status === 'undiscovered' ? 'grey' : 'invalid';
|
||||
const isNew = m.discovered ? '<span class="pill blue">new</span>' : '';
|
||||
return '<div class="probe-row"><span>' + esc(m.id) + '</span>' + isNew +
|
||||
'<span style="flex:1"></span><span class="pill ' + cls + '">' +
|
||||
esc(m.status === 'undiscovered' ? 'listed' : m.status) + '</span></div>';
|
||||
}).join('');
|
||||
|
||||
const limits = Object.entries(r.observedLimits || {});
|
||||
return '<div class="probe-head">' +
|
||||
'<span class="pill ' + (r.auth === 'ok' ? 'ok' : 'invalid') + '">auth ' + esc(r.auth) + '</span>' +
|
||||
'<span class="dim">' + esc(r.keyMasked) + '</span>' +
|
||||
'<span class="dim">' + r.discovered.length + ' discovered · ' + r.ms + 'ms</span>' +
|
||||
'</div>' +
|
||||
(limits.length ? '<div class="bar-window">observed limits: ' +
|
||||
esc(limits.map(([k, v]) => k + '=' + v).join(', ')) + '</div>' : '') +
|
||||
(r.errors.length ? '<div class="err-line">' + esc(r.errors.join('; ')) + '</div>' : '') +
|
||||
rows +
|
||||
(r.discovered.filter(m => !known.includes(m)).length
|
||||
? '<div class="bar-window">' + r.discovered.filter(m => !known.includes(m)).length +
|
||||
' id(s) not yet in providers/' + esc(p.id) + '.json — run scripts/verify-provider.mjs ' + esc(p.id) + ' for a paste-ready block</div>'
|
||||
: '');
|
||||
}).join('') + '</div>';
|
||||
}
|
||||
|
||||
function renderCards() {
|
||||
const el = document.getElementById('cards');
|
||||
if (state.providers.length === 0) {
|
||||
el.innerHTML = '<div class="empty">No providers loaded. Drop a JSON file into <code>providers/</code>.</div>';
|
||||
return;
|
||||
}
|
||||
|
||||
el.innerHTML = state.providers.map(p => {
|
||||
const h = health(p);
|
||||
const total = p.ok + p.fail;
|
||||
const badge = p.limitsSource === 'unverified'
|
||||
? '<span class="pill purple" title="limits could not be verified from provider docs; the guard layer learns the real values from upstream headers">limits unverified</span>'
|
||||
: p.limitsSource === 'observed' ? '<span class="pill blue" title="limits learned from upstream rate-limit headers">limits observed</span>' : '';
|
||||
|
||||
return '<div class="card ' + (p.enabled ? '' : 'off') + ' ' + (p.configured ? '' : 'unconfigured') + '">' +
|
||||
'<div class="card-head">' +
|
||||
'<span class="dot ' + h.dot + '" title="' + esc(h.label) + '"></span>' +
|
||||
'<div class="card-title">' +
|
||||
'<h3>' + esc(p.label) + ' ' + badge + '</h3>' +
|
||||
'<div class="sub">' + esc(p.id) + ' · ' + esc(h.label) + ' · p' + p.priority + '</div>' +
|
||||
'</div>' +
|
||||
'<button class="toggle" data-toggle="' + esc(p.id) + '" aria-pressed="' + (p.enabled ? 'true' : 'false') +
|
||||
'" title="' + (p.enabled ? 'take out of rotation' : 'put back in rotation') + '"></button>' +
|
||||
'</div>' +
|
||||
|
||||
renderAdvice(p) +
|
||||
|
||||
'<div class="metrics">' +
|
||||
'<div><b class="ok-c">' + fmtNum(p.ok) + '</b><span>ok</span></div>' +
|
||||
'<div><b class="' + (p.fail ? 'fail-c' : 'dim') + '">' + fmtNum(p.fail) + '</b><span>failed</span></div>' +
|
||||
'<div><b class="dim">' + (total ? Math.round((p.ok / total) * 100) + '%' : '–') + '</b><span>success</span></div>' +
|
||||
'<div><b class="dim">' + fmtMs(p.latencyMs) + '</b><span>latency ewma</span></div>' +
|
||||
'<div><b class="dim">' + p.models.length + '</b><span>models</span></div>' +
|
||||
'</div>' +
|
||||
|
||||
renderBars(p.bars, p.id) +
|
||||
renderKeys(p) +
|
||||
renderProbe(p) +
|
||||
(p.lastError ? '<div class="err-line">' + esc(p.lastError) + '</div>' : '') +
|
||||
|
||||
'<div class="links">' +
|
||||
(p.docs ? '<a href="' + esc(p.docs) + '" target="_blank" rel="noreferrer noopener">docs ↗</a>' : '') +
|
||||
(p.console ? '<a href="' + esc(p.console) + '" target="_blank" rel="noreferrer noopener">get a key ↗</a>' : '') +
|
||||
'<span style="flex:1"></span>' +
|
||||
(p.configured
|
||||
? '<a href="#" data-probe="' + esc(p.id) + '" title="ask the provider which models this key can reach">probe models</a>' +
|
||||
'<a href="#" data-probe-validate="' + esc(p.id) + '" title="also send a 1-token request per model — costs real quota">deep probe</a>'
|
||||
: '') +
|
||||
'</div>' +
|
||||
'</div>';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function renderModels() {
|
||||
const q = document.getElementById('model-search').value.trim().toLowerCase();
|
||||
const rows = [];
|
||||
for (const p of state.providers) {
|
||||
for (const m of p.models) {
|
||||
const hay = (p.id + ' ' + p.label + ' ' + m.id + ' ' + m.alias.join(' ') + ' ' + m.caps.join(' ')).toLowerCase();
|
||||
if (q && !hay.includes(q)) continue;
|
||||
const live = p.enabled && p.configured && m.enabled;
|
||||
rows.push('<tr>' +
|
||||
'<td class="n">' + esc(p.id) + '</td>' +
|
||||
'<td class="n">' + esc(m.id) + '</td>' +
|
||||
'<td><div class="caps">' + m.alias.map(a => '<span class="pill blue">' + esc(a) + '</span>').join('') + '</div></td>' +
|
||||
'<td><div class="caps">' + m.caps.map(c => '<span class="pill grey">' + esc(c) + '</span>').join('') + '</div></td>' +
|
||||
'<td class="n dim">' + (m.context ? fmtNum(m.context) : '–') + '</td>' +
|
||||
'<td>' + (live
|
||||
? '<span class="pill ok">live</span>'
|
||||
: '<span class="pill unknown">' + (!m.enabled ? 'model off' : !p.configured ? 'no key' : 'provider off') + '</span>') + '</td>' +
|
||||
'</tr>');
|
||||
}
|
||||
}
|
||||
document.getElementById('models-body').innerHTML = rows.length
|
||||
? rows.join('')
|
||||
: '<tr><td colspan="6" class="empty">nothing matches “' + esc(q) + '”</td></tr>';
|
||||
document.getElementById('model-count').textContent = rows.length + ' shown';
|
||||
}
|
||||
|
||||
function renderLog() {
|
||||
const q = document.getElementById('log-search').value.trim().toLowerCase();
|
||||
const rows = [];
|
||||
|
||||
for (const l of state.logs) {
|
||||
const hay = (l.requestedModel + ' ' + l.resolution + ' ' + (l.providerId || '') + ' ' + (l.modelId || '') + ' ' + (l.error || '')).toLowerCase();
|
||||
if (q && !hay.includes(q)) continue;
|
||||
|
||||
const statusClass = l.status >= 500 ? 'status-5xx' : l.status >= 400 ? 'status-4xx' : 'status-200';
|
||||
const hasDetail = (l.attempts && l.attempts.length > 0) || l.error ||
|
||||
(l.preview && (l.preview.prompt || l.preview.response));
|
||||
const open = expanded.has(l.id);
|
||||
|
||||
rows.push('<tr class="' + (hasDetail ? 'clickable' : '') + '" data-log="' + esc(l.id) + '">' +
|
||||
'<td class="n dim">' + fmtTime(l.at) + '</td>' +
|
||||
'<td class="n">' + esc(l.requestedModel) + '</td>' +
|
||||
'<td class="n dim">' + esc(l.resolution) + '</td>' +
|
||||
'<td class="n">' + (l.providerId ? esc(l.providerId) + '<span class="dim">/' + esc(l.modelId) + '</span>' : '<span class="dim">–</span>') + '</td>' +
|
||||
// `attempts` holds only the *failed* tries, so a success is one more than
|
||||
// its length, and a total failure is exactly its length.
|
||||
'<td class="n' + (l.attempts && l.attempts.length ? ' status-4xx' : ' dim') + '">' +
|
||||
(l.status < 400 ? l.attempts.length + 1 : l.attempts.length) +
|
||||
(hasDetail ? (open ? ' ▾' : ' ▸') : '') + '</td>' +
|
||||
'<td class="n ' + statusClass + '">' + l.status + (l.stream ? ' <span class="pill purple">sse</span>' : '') + '</td>' +
|
||||
'<td class="n dim">' + fmtMs(l.ms) + '</td>' +
|
||||
'<td class="n dim">' + (l.tokensIn + l.tokensOut ? fmtNum(l.tokensIn) + '→' + fmtNum(l.tokensOut) : '–') + '</td>' +
|
||||
'</tr>');
|
||||
|
||||
if (open && hasDetail) {
|
||||
const body = l.preview && (l.preview.prompt || l.preview.response)
|
||||
? '<div class="msg-h">prompt</div><div class="msg">' + esc(l.preview.prompt || '(empty)') + '</div>' +
|
||||
'<div class="msg-h">response' +
|
||||
(l.tokensIn + l.tokensOut ? ' · ' + fmtNum(l.tokensIn) + ' in / ' + fmtNum(l.tokensOut) + ' out' : '') +
|
||||
(l.cache ? ' · from ' + esc(l.cache) + ' cache' : '') +
|
||||
'</div><div class="msg out">' + esc(l.preview.response || '(no content)') + '</div>'
|
||||
: '';
|
||||
const lines = (l.attempts || []).map(a =>
|
||||
'<div>· ' + esc(a.providerId) + '/' + esc(a.modelId) +
|
||||
(a.keyId ? ' [' + esc(a.keyId) + ']' : '') +
|
||||
' → <span class="' + (a.status >= 500 ? 'status-5xx' : a.status >= 400 ? 'status-4xx' : 'dim') + '">' +
|
||||
(a.status === null ? 'transport error' : a.status) + '</span> in ' + fmtMs(a.ms) +
|
||||
(a.error ? ' — ' + esc(a.error) : '') + '</div>');
|
||||
if (l.error) lines.push('<div class="status-5xx">· ' + esc(l.error) + '</div>');
|
||||
rows.push('<tr><td colspan="8" class="attempts-detail">' + lines.join('') + body + '</td></tr>');
|
||||
}
|
||||
}
|
||||
|
||||
document.getElementById('log-body').innerHTML = rows.length
|
||||
? rows.join('')
|
||||
: '<tr><td colspan="8" class="empty">no requests yet — send one to <code>/v1/chat/completions</code></td></tr>';
|
||||
}
|
||||
|
||||
function renderStats() {
|
||||
const s = state.stats;
|
||||
if (!s) return;
|
||||
document.getElementById('s-providers').textContent = s.providers.usable + '/' + s.providers.total;
|
||||
document.getElementById('s-models').textContent = s.models.enabled;
|
||||
document.getElementById('s-ok').textContent = fmtNum(s.requests.ok);
|
||||
document.getElementById('s-fail').textContent = fmtNum(s.requests.fail);
|
||||
document.getElementById('s-strategy').textContent = s.strategy;
|
||||
document.getElementById('s-uptime').textContent = fmtDuration(s.uptimeSec);
|
||||
}
|
||||
|
||||
function renderAlerts() {
|
||||
const section = document.getElementById('alerts-section');
|
||||
const el = document.getElementById('alerts');
|
||||
const alerts = (state.alerts && state.alerts.alerts) || [];
|
||||
|
||||
if (alerts.length === 0) { section.classList.add('hide'); el.innerHTML = ''; return; }
|
||||
section.classList.remove('hide');
|
||||
|
||||
el.innerHTML = alerts.map(a =>
|
||||
'<div class="alert ' + esc(a.level) + '">' +
|
||||
'<span class="src">' + esc(a.source) + '</span>' +
|
||||
'<span>' + esc(a.message) + '</span>' +
|
||||
'</div>').join('');
|
||||
}
|
||||
|
||||
/** Per-limit trajectory, so a bar at 40% says whether that is fine or not. */
|
||||
function forecastFor(providerId, key) {
|
||||
const list = (state.alerts && state.alerts.forecasts) || [];
|
||||
const p = list.find(f => f.providerId === providerId);
|
||||
if (!p) return null;
|
||||
return p.forecasts.find(f => f.key === key) || null;
|
||||
}
|
||||
|
||||
function renderUsage() {
|
||||
const u = state.usage;
|
||||
if (!u) return;
|
||||
document.getElementById('usage-window').textContent = '· ' + u.window;
|
||||
|
||||
const t = u.totals;
|
||||
const card = (big, lbl, sub, cls) =>
|
||||
'<div class="ucard"><div class="big ' + (cls || '') + '">' + big + '</div>' +
|
||||
'<div class="lbl">' + lbl + '</div>' + (sub ? '<div class="sub">' + sub + '</div>' : '') + '</div>';
|
||||
|
||||
document.getElementById('usage-cards').innerHTML =
|
||||
card(fmtNum(t.tokensIn + t.tokensOut), 'tokens total', fmtNum(t.tokensIn) + ' in · ' + fmtNum(t.tokensOut) + ' out') +
|
||||
card(fmtNum(t.requests), 'requests', t.ok + ' ok · ' + t.failed + ' failed', t.failed ? '' : 'ok-c') +
|
||||
card(t.avgMs + 'ms', 'avg latency', t.streamed + ' streamed') +
|
||||
// Cache hits are the interesting number: those are answers that cost nothing.
|
||||
card(fmtNum(t.cached), 'served from cache',
|
||||
t.requests ? Math.round((t.cached / t.requests) * 100) + '% of requests, 0 quota' : '', t.cached ? 'ok-c' : 'dim');
|
||||
|
||||
const rows = (list, name) => list.length === 0
|
||||
? '<div class="brow"><span class="dim">no traffic yet</span><span></span></div>'
|
||||
: list.map(x => '<div class="brow">' +
|
||||
'<span class="n">' + esc(name(x)) + '</span>' +
|
||||
'<span class="n">' + fmtNum(x.tokensIn + x.tokensOut) + ' tok <span class="dim">· ' + x.requests + '×</span></span>' +
|
||||
'</div>').join('');
|
||||
|
||||
document.getElementById('by-provider').innerHTML = rows(u.byProvider, x => x.id);
|
||||
document.getElementById('by-model').innerHTML = rows(u.byModel, x => x.id);
|
||||
}
|
||||
|
||||
function renderAll() {
|
||||
renderStats();
|
||||
renderUsage();
|
||||
renderAlerts();
|
||||
renderBanners();
|
||||
renderCards();
|
||||
renderModels();
|
||||
renderLog();
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------- polling
|
||||
|
||||
async function refresh() {
|
||||
try {
|
||||
// Once the event stream is up the log is authoritative on its own, so the
|
||||
// poll drops it and only re-reads state that changes continuously.
|
||||
const wants = [api('/api/stats'), api('/api/providers'), api('/api/alerts'), api('/api/usage')];
|
||||
if (!liveConnected) wants.push(api('/api/logs?limit=200'));
|
||||
const [stats, providers, alerts, usage, logs] = await Promise.all(wants);
|
||||
|
||||
state.stats = stats;
|
||||
state.providers = providers.providers;
|
||||
state.issues = providers.issues || [];
|
||||
state.alerts = alerts;
|
||||
state.usage = usage;
|
||||
if (logs) state.logs = logs.logs;
|
||||
lastOk = Date.now();
|
||||
showApp();
|
||||
renderAll();
|
||||
setPulse('ok');
|
||||
if (!liveAbort) connectLive();
|
||||
} catch (err) {
|
||||
if (err && err.unauthorized) { showGate(); return; }
|
||||
setPulse(Date.now() - lastOk > 15000 ? 'dead' : 'stale');
|
||||
}
|
||||
}
|
||||
|
||||
async function probeProvider(id, validate) {
|
||||
state.probes[id] = { loading: true };
|
||||
renderCards();
|
||||
try {
|
||||
const res = await api('/api/providers/' + encodeURIComponent(id) + '/probe', {
|
||||
method: 'POST', body: JSON.stringify({ validate: !!validate }),
|
||||
});
|
||||
state.probes[id] = { results: res.results };
|
||||
} catch (err) {
|
||||
state.probes[id] = { error: String(err && err.message || err) };
|
||||
}
|
||||
renderCards();
|
||||
}
|
||||
|
||||
function setPulse(kind) {
|
||||
const el = document.getElementById('pulse');
|
||||
el.className = 'pulse' + (kind === 'ok' ? '' : ' ' + kind);
|
||||
document.getElementById('pulse-text').textContent =
|
||||
kind === 'ok' ? 'live' : kind === 'stale' ? 'retrying' : 'disconnected';
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
clearInterval(timer);
|
||||
// Polling a hidden tab burns CPU for nobody; the visibilitychange handler
|
||||
// brings it straight back with a fresh fetch.
|
||||
timer = setInterval(() => { if (!paused && !document.hidden) refresh(); }, POLL_MS);
|
||||
}
|
||||
|
||||
/* ---- live request log -------------------------------------------------- */
|
||||
/* Provider meters and cooldowns change continuously so they stay polled, but a
|
||||
request is a discrete event and should appear the instant it happens rather
|
||||
than up to POLL_MS later. Read with fetch rather than EventSource so the
|
||||
virtual key travels in a header instead of a query string. */
|
||||
|
||||
let liveAbort = null;
|
||||
let liveBackoff = 1000;
|
||||
let liveConnected = false;
|
||||
|
||||
async function connectLive() {
|
||||
if (liveAbort) liveAbort.abort();
|
||||
liveAbort = new AbortController();
|
||||
|
||||
try {
|
||||
const headers = { accept: 'text/event-stream' };
|
||||
if (apiKey) headers['authorization'] = 'Bearer ' + apiKey;
|
||||
const res = await fetch('/api/events', { headers, signal: liveAbort.signal });
|
||||
if (res.status === 401 || res.status === 403) { showGate(); return; }
|
||||
if (!res.ok || !res.body) throw new Error('events ' + res.status);
|
||||
|
||||
liveBackoff = 1000;
|
||||
setLiveState(true);
|
||||
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let event = 'message';
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
let nl;
|
||||
while ((nl = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, nl).replace(/\r$/, '');
|
||||
buffer = buffer.slice(nl + 1);
|
||||
if (line.startsWith('event:')) { event = line.slice(6).trim(); continue; }
|
||||
if (line.startsWith('data:')) {
|
||||
const payload = line.slice(5).trim();
|
||||
if (event === 'log') {
|
||||
try { onLiveLog(JSON.parse(payload)); } catch { /* malformed frame */ }
|
||||
}
|
||||
event = 'message';
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err && err.name === 'AbortError') return;
|
||||
}
|
||||
|
||||
setLiveState(false);
|
||||
// Reconnect with backoff. The polled sections keep the page useful meanwhile.
|
||||
liveBackoff = Math.min(liveBackoff * 2, 15000);
|
||||
setTimeout(() => { if (!paused) connectLive(); }, liveBackoff);
|
||||
}
|
||||
|
||||
function onLiveLog(entry) {
|
||||
// Prepend rather than refetch: the entry is already the whole truth.
|
||||
state.logs.unshift(entry);
|
||||
if (state.logs.length > 200) state.logs.length = 200;
|
||||
flashLive();
|
||||
renderLog();
|
||||
}
|
||||
|
||||
function setLiveState(on) {
|
||||
liveConnected = on;
|
||||
const el = document.getElementById('live-badge');
|
||||
el.textContent = on ? 'streaming' : 'polling';
|
||||
el.className = 'pill ' + (on ? 'ok' : 'grey');
|
||||
}
|
||||
|
||||
let flashTimer = null;
|
||||
function flashLive() {
|
||||
const el = document.getElementById('live-badge');
|
||||
el.style.transform = 'scale(1.14)';
|
||||
clearTimeout(flashTimer);
|
||||
flashTimer = setTimeout(() => { el.style.transform = ''; }, 180);
|
||||
}
|
||||
|
||||
function showGate() {
|
||||
document.getElementById('gate').classList.remove('hide');
|
||||
document.getElementById('app').classList.add('hide');
|
||||
clearInterval(timer);
|
||||
}
|
||||
function showApp() {
|
||||
document.getElementById('gate').classList.add('hide');
|
||||
document.getElementById('app').classList.remove('hide');
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------- events
|
||||
|
||||
document.addEventListener('click', async (e) => {
|
||||
const toggle = e.target.closest('[data-toggle]');
|
||||
if (toggle) {
|
||||
const id = toggle.dataset.toggle;
|
||||
const next = toggle.getAttribute('aria-pressed') !== 'true';
|
||||
toggle.setAttribute('aria-pressed', String(next)); // optimistic
|
||||
try {
|
||||
await api('/api/providers/' + encodeURIComponent(id) + '/enabled', { method: 'POST', body: JSON.stringify({ enabled: next }) });
|
||||
} catch { /* the next poll re-syncs from the server */ }
|
||||
refresh();
|
||||
return;
|
||||
}
|
||||
|
||||
const copy = e.target.closest('[data-copy]');
|
||||
if (copy) {
|
||||
navigator.clipboard?.writeText(copy.dataset.copy);
|
||||
const was = copy.textContent;
|
||||
copy.textContent = 'copied';
|
||||
setTimeout(() => { copy.textContent = was; }, 1200);
|
||||
return;
|
||||
}
|
||||
|
||||
const save = e.target.closest('[data-key-save]');
|
||||
if (save) {
|
||||
const id = save.dataset.keySave;
|
||||
const input = document.querySelector('[data-key-input="' + CSS.escape(id) + '"]');
|
||||
const value = input && input.value.trim();
|
||||
if (!value) return;
|
||||
save.disabled = true; save.textContent = 'saving…';
|
||||
try {
|
||||
await api('/api/providers/' + encodeURIComponent(id) + '/key', { method: 'POST', body: JSON.stringify({ key: value }) });
|
||||
if (input) input.value = '';
|
||||
await refresh();
|
||||
probeProvider(id, false); // immediately show what the new key can reach
|
||||
} catch (err) {
|
||||
save.textContent = 'failed';
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const probe = e.target.closest('[data-probe]');
|
||||
if (probe) { e.preventDefault(); probeProvider(probe.dataset.probe, false); return; }
|
||||
const deep = e.target.closest('[data-probe-validate]');
|
||||
if (deep) { e.preventDefault(); probeProvider(deep.dataset.probeValidate, true); return; }
|
||||
|
||||
const row = e.target.closest('[data-log]');
|
||||
if (row) {
|
||||
const id = row.dataset.log;
|
||||
if (expanded.has(id)) expanded.delete(id); else expanded.add(id);
|
||||
renderLog();
|
||||
}
|
||||
});
|
||||
|
||||
document.getElementById('btn-reload').addEventListener('click', async (e) => {
|
||||
e.target.disabled = true;
|
||||
try { await api('/api/reload', { method: 'POST', body: '{}' }); } catch { /* surfaced by the pulse */ }
|
||||
await refresh();
|
||||
e.target.disabled = false;
|
||||
});
|
||||
|
||||
document.getElementById('btn-pause').addEventListener('click', (e) => {
|
||||
paused = !paused;
|
||||
e.target.textContent = paused ? 'resume' : 'pause';
|
||||
e.target.classList.toggle('on', paused);
|
||||
if (paused) {
|
||||
if (liveAbort) liveAbort.abort();
|
||||
liveAbort = null;
|
||||
setLiveState(false);
|
||||
} else {
|
||||
refresh();
|
||||
connectLive();
|
||||
}
|
||||
});
|
||||
|
||||
document.getElementById('model-search').addEventListener('input', renderModels);
|
||||
document.getElementById('log-search').addEventListener('input', renderLog);
|
||||
|
||||
document.getElementById('gate-go').addEventListener('click', async () => {
|
||||
const value = document.getElementById('gate-key').value.trim();
|
||||
if (!value) return;
|
||||
apiKey = value;
|
||||
try {
|
||||
await api('/api/stats');
|
||||
localStorage.setItem(KEY_STORAGE, apiKey);
|
||||
document.getElementById('gate-err').classList.add('hide');
|
||||
schedule();
|
||||
refresh();
|
||||
} catch {
|
||||
apiKey = '';
|
||||
const err = document.getElementById('gate-err');
|
||||
err.textContent = 'that key was rejected';
|
||||
err.classList.remove('hide');
|
||||
}
|
||||
});
|
||||
document.getElementById('gate-key').addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Enter') document.getElementById('gate-go').click();
|
||||
});
|
||||
|
||||
document.addEventListener('visibilitychange', () => { if (!document.hidden && !paused) refresh(); });
|
||||
|
||||
schedule();
|
||||
refresh();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Entry point: wire the registry, store and router together and listen.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { join, resolve } from 'node:path';
|
||||
|
||||
import { ContextStore, ExactCache, KeyVault, Registry, Router, SemanticCache, Store, Tokenizer, consoleLogger, loadConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from './server.ts';
|
||||
import { isLoopback } from './http.ts';
|
||||
|
||||
const logger = consoleLogger;
|
||||
|
||||
const root = resolve(process.env['FREEWAY_ROOT'] ?? process.cwd());
|
||||
const providersDir = resolve(process.env['FREEWAY_PROVIDERS'] ?? join(root, 'providers'));
|
||||
const dataDir = resolve(process.env['FREEWAY_DATA'] ?? join(root, 'data'));
|
||||
const configPath = resolve(process.env['FREEWAY_CONFIG'] ?? join(root, 'freeway.config.json'));
|
||||
|
||||
loadDotEnv(join(root, '.env'));
|
||||
|
||||
const { config, issues: configIssues } = loadConfig(configPath, process.env);
|
||||
for (const issue of configIssues) {
|
||||
logger.warn(`config ${issue.path ? `${issue.path}: ` : ''}${issue.message}`);
|
||||
}
|
||||
|
||||
// Accepting pasted keys is safe on loopback; on any other interface it needs an
|
||||
// admin key, or anyone who can reach the port could add credentials.
|
||||
const vault = new KeyVault({
|
||||
file: join(dataDir, 'keys.json'),
|
||||
logger,
|
||||
writable: isLoopback(config.host) || config.adminKey !== null,
|
||||
});
|
||||
|
||||
const registry = new Registry({ dir: providersDir, env: process.env, logger, extraKeys: vault.all() });
|
||||
const store = new Store({ file: join(dataDir, 'usage.json'), cooldownMs: config.cooldownSeconds * 1000, logger });
|
||||
const router = new Router(registry, store, config);
|
||||
|
||||
// An open gateway on a public interface hands every provider key you own to
|
||||
// anyone who can reach the port. Refusing to bind is the only safe default.
|
||||
if (config.keys.length === 0) {
|
||||
if (!isLoopback(config.host)) {
|
||||
logger.error(
|
||||
`refusing to bind ${config.host}:${config.port} with no virtual keys configured.\n` +
|
||||
` Anyone who can reach that address could spend every provider quota you have.\n` +
|
||||
` Fix: set FREEWAY_KEYS=fw-<something-random> (or "keys" in ${configPath}), or bind 127.0.0.1.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
logger.warn('no virtual keys configured — the gateway is OPEN to anything that can reach localhost.');
|
||||
logger.warn(' set FREEWAY_KEYS=fw-… before exposing this beyond your own machine.');
|
||||
}
|
||||
|
||||
// Conversation history lives in sqlite so a thread survives both a restart and
|
||||
// a failover to a model with a much smaller window.
|
||||
const contextStore = config.context.enabled ? new ContextStore(join(dataDir, 'freeway.db')) : undefined;
|
||||
const tokenizer = new Tokenizer();
|
||||
|
||||
const cache = config.cache.mode !== 'off'
|
||||
? new ExactCache({ file: join(dataDir, 'cache.db'), mode: config.cache.mode, ttlMs: config.cache.ttlMs, maxEntries: config.cache.maxEntries })
|
||||
: undefined;
|
||||
const semanticCache = config.cache.semantic.enabled
|
||||
? new SemanticCache({
|
||||
enabled: true,
|
||||
threshold: config.cache.semantic.threshold,
|
||||
maxEntries: config.cache.semantic.maxEntries,
|
||||
ttlMs: config.cache.ttlMs,
|
||||
})
|
||||
: undefined;
|
||||
|
||||
// Exposed gateways get real defaults. Leaving every limit null is fine on
|
||||
// loopback, where the only caller is you; on any other interface it means a
|
||||
// single leaked key can drain every free tier you own before you notice.
|
||||
const exposed = !isLoopback(config.host);
|
||||
if (exposed) {
|
||||
const g = config.guard;
|
||||
if (g.perKey.rpm === null) g.perKey.rpm = 120;
|
||||
if (g.perKey.concurrency === null) g.perKey.concurrency = 8;
|
||||
if (g.perIp.rpm === null) g.perIp.rpm = 240;
|
||||
if (g.perIp.concurrency === null) g.perIp.concurrency = 16;
|
||||
logger.info(`bound to ${config.host} — applied default inbound limits (perKey ${g.perKey.rpm}/min, perIp ${g.perIp.rpm}/min)`);
|
||||
|
||||
if (config.adminKey === null) {
|
||||
logger.warn('no FREEWAY_ADMIN_KEY set — /api/* is refused entirely on a non-loopback bind.');
|
||||
logger.warn(' Set one to use the dashboard remotely: node -e "console.log(\'fw-admin-\'+crypto.randomUUID())"');
|
||||
}
|
||||
if (config.corsOrigins.length === 0) {
|
||||
logger.info('no corsOrigins configured — browsers on other origins cannot call this gateway (this is the safe default).');
|
||||
}
|
||||
}
|
||||
|
||||
const stopAutosave = store.startAutosave();
|
||||
const server = createGateway({
|
||||
registry, store, router, config, logger, vault,
|
||||
...(contextStore ? { contextStore, tokenizer } : {}),
|
||||
...(cache ? { cache } : {}),
|
||||
...(semanticCache ? { semanticCache } : {}),
|
||||
onReload: () => router.setConfig(config),
|
||||
});
|
||||
|
||||
server.listen(config.port, config.host, () => {
|
||||
const providers = registry.all();
|
||||
const usable = registry.usable();
|
||||
logger.info(`listening on http://${config.host}:${config.port}`);
|
||||
logger.info(` dashboard http://${config.host}:${config.port}/`);
|
||||
logger.info(` OpenAI base http://${config.host}:${config.port}/v1`);
|
||||
logger.info(`strategy=${config.strategy} providers=${providers.length} usable=${usable.length} models=${registry.allModels().length}`);
|
||||
|
||||
for (const p of providers) {
|
||||
if (p.configured) {
|
||||
logger.info(` ✓ ${p.id} — ${p.keys.length} key${p.keys.length === 1 ? '' : 's'}, ${p.models.length} models`);
|
||||
} else {
|
||||
logger.info(` ○ ${p.id} — ${p.configError ?? 'not configured'}`);
|
||||
}
|
||||
}
|
||||
for (const issue of registry.issues()) {
|
||||
logger[issue.level === 'error' ? 'warn' : 'info'](` ${issue.level}: ${issue.file}: ${issue.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
function shutdown(signal: string): void {
|
||||
logger.info(`${signal} received, shutting down`);
|
||||
stopAutosave();
|
||||
store.save();
|
||||
contextStore?.close();
|
||||
cache?.close();
|
||||
server.close(() => process.exit(0));
|
||||
// Do not let a hung keep-alive connection block a container restart.
|
||||
setTimeout(() => process.exit(0), 5000).unref();
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
|
||||
/**
|
||||
* Minimal `.env` reader.
|
||||
*
|
||||
* A runtime dotenv dependency would be the first crack in the zero-dependency
|
||||
* rule, and the format we actually need is three lines of parsing.
|
||||
*/
|
||||
function loadDotEnv(path: string): void {
|
||||
if (!existsSync(path)) return;
|
||||
let text: string;
|
||||
try {
|
||||
text = readFileSync(path, 'utf8');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const line of text.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const eq = trimmed.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
const key = trimmed.slice(0, eq).trim().replace(/^export\s+/, '');
|
||||
let value = trimmed.slice(eq + 1).trim();
|
||||
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
// A real environment variable always wins over the file.
|
||||
if (process.env[key] === undefined) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/** Thin helpers over node:http. No framework, no middleware stack. */
|
||||
|
||||
import { createHash, timingSafeEqual } from 'node:crypto';
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http';
|
||||
|
||||
export type Req = IncomingMessage;
|
||||
export type Res = ServerResponse;
|
||||
|
||||
export interface BodyResult {
|
||||
ok: boolean;
|
||||
raw: string;
|
||||
/** Set when the body exceeded the cap or was not valid JSON. */
|
||||
error: string | null;
|
||||
tooLarge: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a request body with a hard ceiling.
|
||||
*
|
||||
* The cap is enforced while reading rather than after, so a client cannot
|
||||
* exhaust memory by streaming gigabytes at a gateway that only checks the size
|
||||
* once the whole thing has arrived.
|
||||
*/
|
||||
export function readBody(req: Req, maxBytes: number): Promise<BodyResult> {
|
||||
return new Promise((resolve) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
let settled = false;
|
||||
|
||||
const finish = (result: BodyResult): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
req.on('data', (chunk: Buffer) => {
|
||||
size += chunk.length;
|
||||
if (size > maxBytes) {
|
||||
// Pause rather than destroy: killing the socket here would take the 413
|
||||
// response down with it, and the caller would see a connection reset
|
||||
// instead of being told what was wrong. The caller closes the socket
|
||||
// once the response has flushed.
|
||||
req.pause();
|
||||
finish({ ok: false, raw: '', error: `request body exceeds ${maxBytes} bytes`, tooLarge: true });
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
req.on('end', () => finish({ ok: true, raw: Buffer.concat(chunks).toString('utf8'), error: null, tooLarge: false }));
|
||||
req.on('error', (err) => finish({ ok: false, raw: '', error: err.message, tooLarge: false }));
|
||||
});
|
||||
}
|
||||
|
||||
export function sendJson(res: Res, status: number, body: unknown, headers: Record<string, string> = {}): void {
|
||||
if (res.headersSent) return;
|
||||
const payload = JSON.stringify(body);
|
||||
res.writeHead(status, {
|
||||
'content-type': 'application/json; charset=utf-8',
|
||||
'content-length': String(Buffer.byteLength(payload)),
|
||||
...headers,
|
||||
});
|
||||
res.end(payload);
|
||||
}
|
||||
|
||||
export function sendText(res: Res, status: number, body: string, contentType = 'text/plain; charset=utf-8'): void {
|
||||
if (res.headersSent) return;
|
||||
res.writeHead(status, { 'content-type': contentType, 'content-length': String(Buffer.byteLength(body)) });
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
export interface ApiError {
|
||||
message: string;
|
||||
type: string;
|
||||
code: string | null;
|
||||
}
|
||||
|
||||
/** OpenAI-shaped error envelope, plus a `freeway` block for our own diagnostics. */
|
||||
export function sendError(
|
||||
res: Res,
|
||||
status: number,
|
||||
error: ApiError,
|
||||
extra: Record<string, unknown> = {},
|
||||
headers: Record<string, string> = {},
|
||||
): void {
|
||||
sendJson(res, status, { error, ...(Object.keys(extra).length > 0 ? { freeway: extra } : {}) }, headers);
|
||||
}
|
||||
|
||||
export function clientIp(req: Req): string {
|
||||
// Trusting XFF blindly lets a client spoof its own identity, so only the
|
||||
// socket address is authoritative unless the operator puts a proxy in front.
|
||||
return req.socket.remoteAddress ?? 'unknown';
|
||||
}
|
||||
|
||||
export function bearerToken(req: Req): string | null {
|
||||
const auth = req.headers.authorization;
|
||||
if (typeof auth === 'string') {
|
||||
const m = /^Bearer\s+(.+)$/i.exec(auth.trim());
|
||||
if (m?.[1]) return m[1].trim();
|
||||
}
|
||||
const alt = req.headers['x-api-key'];
|
||||
if (typeof alt === 'string' && alt.trim()) return alt.trim();
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isLoopback(host: string): boolean {
|
||||
return host === '127.0.0.1' || host === 'localhost' || host === '::1' || host === '::ffff:127.0.0.1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Constant-time comparison of two secrets.
|
||||
*
|
||||
* `timingSafeEqual` needs equal-length buffers, and returning early on a length
|
||||
* mismatch would leak the key's length. Hashing both sides first gives fixed
|
||||
* width regardless of input, so neither length nor content is observable.
|
||||
*/
|
||||
export function safeEqual(a: string, b: string): boolean {
|
||||
const ha = createHash('sha256').update(a).digest();
|
||||
const hb = createHash('sha256').update(b).digest();
|
||||
return timingSafeEqual(ha, hb);
|
||||
}
|
||||
@@ -0,0 +1,724 @@
|
||||
/**
|
||||
* The failover loop: walk candidates until one answers, then account for it.
|
||||
*
|
||||
* The rule that shapes this file: **once a byte of a stream has been written to
|
||||
* the client, failover is over**. A half-delivered SSE stream cannot be taken
|
||||
* back, so the commit point sits after the upstream returns 200 headers but
|
||||
* before its body is piped. A 429 arriving at that moment is still recoverable;
|
||||
* a 429 arriving one byte later is not.
|
||||
*/
|
||||
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import type { ReadableStream as WebReadableStream } from 'node:stream/web';
|
||||
|
||||
import {
|
||||
ContextTooLargeError,
|
||||
SemanticCache,
|
||||
cacheKey,
|
||||
creditsFor,
|
||||
describeRefit,
|
||||
isCacheable,
|
||||
queryTextOf,
|
||||
refit,
|
||||
replayAsStream,
|
||||
systemPromptOf,
|
||||
type Blocked,
|
||||
type Candidate,
|
||||
type CacheEntry,
|
||||
type CacheTier,
|
||||
type ContextStore,
|
||||
type ExactCache,
|
||||
type FreewayConfig,
|
||||
type LimitSpec,
|
||||
type Logger,
|
||||
type Registry,
|
||||
type RefitMessage,
|
||||
type Router,
|
||||
type Store,
|
||||
previewOf,
|
||||
type Tokenizer,
|
||||
} from '@freeway/core';
|
||||
|
||||
import { sendError, sendJson, type Res } from './http.ts';
|
||||
import type { RouteRequest } from '@freeway/core';
|
||||
import {
|
||||
callUpstream,
|
||||
completionText,
|
||||
deadline,
|
||||
estimateTokens,
|
||||
retryAfterMs,
|
||||
usageFromBody,
|
||||
type TokenUsage,
|
||||
} from './upstream.ts';
|
||||
import { UsageSniffer } from './sniffer.ts';
|
||||
import { recordReply } from './session.ts';
|
||||
|
||||
export interface ProxyDeps {
|
||||
registry: Registry;
|
||||
store: Store;
|
||||
router: Router;
|
||||
config: FreewayConfig;
|
||||
logger: Logger;
|
||||
guard?: {
|
||||
limiter: { recordTokens(keyId: string | null, ip: string, tokens: number): void };
|
||||
calibrator: { observe(providerId: string, declared: LimitSpec, headers: Headers): unknown };
|
||||
};
|
||||
context?: {
|
||||
store: ContextStore;
|
||||
tokenizer: Tokenizer;
|
||||
/** Summarises a dropped span, routed back through this same gateway. */
|
||||
summarize?: (messages: RefitMessage[], budgetTokens: number) => Promise<string | null>;
|
||||
};
|
||||
cache?: {
|
||||
exact: ExactCache;
|
||||
semantic?: SemanticCache;
|
||||
/** Embeds a query for the semantic tier, through this same gateway. */
|
||||
embed?: (text: string) => Promise<number[] | null>;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ProxyRequest {
|
||||
route: RouteRequest;
|
||||
path: string;
|
||||
body: Record<string, unknown>;
|
||||
stream: boolean;
|
||||
/** Prompt text, used only to estimate tokens when a provider reports none. */
|
||||
promptText: string;
|
||||
requestedModel: string;
|
||||
/** Who asked, so inbound token limits can be charged to them. */
|
||||
caller?: { keyId: string | null; ip: string };
|
||||
/** Content-scan findings, recorded on the log entry. */
|
||||
findings?: { category: string; rule: string; path: string; severity: string }[];
|
||||
/** Set when the request carried a `session` id. */
|
||||
session?: { id: string; baseSeq: number; restored: number } | null;
|
||||
/** Full conversation after hydration, before any refit. */
|
||||
history?: RefitMessage[];
|
||||
}
|
||||
|
||||
/** Used when a provider does not publish a model's window. Deliberately small. */
|
||||
const DEFAULT_CONTEXT = 8192;
|
||||
|
||||
interface AttemptRecord {
|
||||
providerId: string;
|
||||
modelId: string;
|
||||
keyId: string | null;
|
||||
status: number | null;
|
||||
ms: number;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
export async function proxy(deps: ProxyDeps, req: ProxyRequest, res: Res): Promise<void> {
|
||||
const { store, router, config, logger } = deps;
|
||||
const startedAt = Date.now();
|
||||
const requestId = store.nextRequestId();
|
||||
|
||||
const result = router.route(req.route);
|
||||
|
||||
// ---- cache ---------------------------------------------------------------
|
||||
// Checked after routing so the key can use the *resolution* rather than the
|
||||
// raw model string: `auto` and `fast` pointing at the same models should share
|
||||
// an entry. A hit costs no quota and no upstream call.
|
||||
const cacheable = deps.cache !== undefined && result.matched && isCacheable(deps.cache.exact.mode, req.body);
|
||||
let entryKey: string | null = null;
|
||||
let semanticScope: string | null = null;
|
||||
let queryText = '';
|
||||
/** Held from the miss path so a stored answer can be indexed without re-embedding. */
|
||||
let pendingEmbedding: number[] | null = null;
|
||||
|
||||
if (cacheable && deps.cache) {
|
||||
entryKey = cacheKey(result.resolution, req.body);
|
||||
const hit = deps.cache.exact.get(entryKey);
|
||||
if (hit) {
|
||||
serveFromCache(deps, res, req, hit, 'exact', 1, result.resolution, requestId, startedAt);
|
||||
return;
|
||||
}
|
||||
|
||||
const semantic = deps.cache.semantic;
|
||||
if (semantic?.enabled && deps.cache.embed && Array.isArray(req.body['messages'])) {
|
||||
const messages = req.body['messages'] as { role?: string; content?: unknown }[];
|
||||
queryText = queryTextOf(messages);
|
||||
semanticScope = SemanticCache.scopeOf(result.resolution, systemPromptOf(messages));
|
||||
|
||||
if (queryText && !semantic.recentlyMissed(semanticScope, queryText)) {
|
||||
const embedding = await deps.cache.embed(queryText);
|
||||
if (embedding) {
|
||||
const match = semantic.search(embedding, semanticScope);
|
||||
if (match) {
|
||||
const entry = deps.cache.exact.get(match.entry.cacheKey);
|
||||
if (entry) {
|
||||
serveFromCache(deps, res, req, entry, 'semantic', match.score, result.resolution, requestId, startedAt);
|
||||
return;
|
||||
}
|
||||
}
|
||||
semantic.noteMiss(semanticScope, queryText);
|
||||
// Remember the vector so a hit can be stored once the answer arrives.
|
||||
pendingEmbedding = embedding;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Nothing matched the model string at all — that is a 404, not a capacity problem.
|
||||
if (!result.matched) {
|
||||
sendError(
|
||||
res,
|
||||
404,
|
||||
{ message: `unknown model "${req.requestedModel}"`, type: 'invalid_request_error', code: 'model_not_found' },
|
||||
{ requested: req.requestedModel, resolution: result.resolution, suggestions: result.suggestions },
|
||||
routeHeaders({ resolution: result.resolution, attempts: 0, ms: Date.now() - startedAt }),
|
||||
);
|
||||
logFailure(deps, requestId, req, result.resolution, [], 404, Date.now() - startedAt, `unknown model "${req.requestedModel}"`);
|
||||
return;
|
||||
}
|
||||
|
||||
// The model exists but every provider offering it is blocked. The blocked list
|
||||
// *is* the error message, so it goes out in full.
|
||||
if (result.candidates.length === 0) {
|
||||
respondAllBlocked(deps, res, req, result.blocked, result.resolution, requestId, startedAt);
|
||||
return;
|
||||
}
|
||||
|
||||
const attempts: AttemptRecord[] = [];
|
||||
const maxAttempts = Math.max(1, config.maxAttempts);
|
||||
const timeout = req.stream ? config.streamTimeoutMs : config.timeoutMs;
|
||||
|
||||
// Candidates were chosen before the first call, so a provider benched partway
|
||||
// through this request is still sitting in the list with its other keys. A
|
||||
// provider-fault means every key will fail identically — skipping them spends
|
||||
// the attempt budget on a provider that might actually answer.
|
||||
const benched = new Set<string>();
|
||||
|
||||
for (const candidate of result.candidates) {
|
||||
if (attempts.length >= maxAttempts) break;
|
||||
const { provider, model, key } = candidate;
|
||||
if (benched.has(provider.id)) continue;
|
||||
const keyId = provider.spec.auth.type === 'none' ? null : key.id;
|
||||
const dl = deadline(timeout);
|
||||
|
||||
// Refit happens HERE, per candidate — not once up front. Each candidate has
|
||||
// a different context window, so a history that fits mistral's 128k has to
|
||||
// be reshaped again when the request fails over to an 8k model.
|
||||
let outgoing = req.body;
|
||||
let contextNote: string | null = null;
|
||||
|
||||
if (req.history && req.history.length > 0) {
|
||||
try {
|
||||
const fitted = await refit(req.history, {
|
||||
contextWindow: model.spec.context ?? DEFAULT_CONTEXT,
|
||||
reserveOutputTokens: config.context.reserveOutputTokens,
|
||||
maxTurns: config.context.maxTurns,
|
||||
model: model.spec.id,
|
||||
...(deps.context ? { tokenizer: deps.context.tokenizer } : {}),
|
||||
...(deps.context?.summarize ? { summarize: deps.context.summarize } : {}),
|
||||
});
|
||||
outgoing = { ...req.body, messages: fitted.messages };
|
||||
contextNote = describeRefit(fitted);
|
||||
} catch (err) {
|
||||
if (err instanceof ContextTooLargeError) {
|
||||
// No trimming can help: the newest message alone exceeds the window.
|
||||
// Say so rather than silently mangling what the caller asked.
|
||||
sendError(res, 413,
|
||||
{ message: err.message, type: 'invalid_request_error', code: 'context_too_large' },
|
||||
{ provider: provider.id, model: model.spec.id, required: err.required, available: err.available },
|
||||
routeHeaders({ provider: provider.id, model: model.spec.id, resolution: result.resolution, attempts: attempts.length + 1, ms: Date.now() - startedAt }),
|
||||
);
|
||||
logFailure(deps, requestId, req, result.resolution, attempts, 413, Date.now() - startedAt, err.message);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const call = await callUpstream({
|
||||
candidate,
|
||||
path: req.path,
|
||||
body: { ...outgoing, model: model.spec.id },
|
||||
signal: dl.signal,
|
||||
});
|
||||
|
||||
// ---- transport failure -------------------------------------------------
|
||||
if (call.response === null) {
|
||||
dl.cancel();
|
||||
const message = call.transportError ?? 'upstream call failed';
|
||||
store.recordFailure(provider.id, keyId, { status: null, message });
|
||||
attempts.push({ providerId: provider.id, modelId: model.spec.id, keyId, status: null, ms: call.ms, error: message });
|
||||
benched.add(provider.id);
|
||||
logger.warn(`${provider.id}/${model.spec.id} ${message}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Whatever the provider just said about its own limits is worth more than
|
||||
// whatever the JSON file claims. Observed on every reply, including errors —
|
||||
// a 429 usually carries the most accurate numbers of all.
|
||||
deps.guard?.calibrator.observe(provider.id, provider.spec.limits, call.response.headers);
|
||||
|
||||
// ---- upstream error ----------------------------------------------------
|
||||
if (!call.ok) {
|
||||
dl.cancel();
|
||||
const status = call.status ?? 0;
|
||||
const errorBody = await safeJson(call.response);
|
||||
const message = extractMessage(errorBody) ?? `${status} from ${provider.id}`;
|
||||
const kind = store.recordFailure(provider.id, keyId, {
|
||||
status,
|
||||
message,
|
||||
// A provider telling us when to come back beats our configured guess.
|
||||
retryAfterMs: retryAfterMs(call.response) ?? config.cooldownSeconds * 1000,
|
||||
});
|
||||
attempts.push({ providerId: provider.id, modelId: model.spec.id, keyId, status, ms: call.ms, error: message });
|
||||
|
||||
// A malformed request fails identically everywhere. Retrying it wastes
|
||||
// quota on a guaranteed failure, so it goes straight back to the caller.
|
||||
if (kind === 'request') {
|
||||
sendJson(res, status, errorBody ?? { error: { message, type: 'invalid_request_error', code: null } }, routeHeaders({
|
||||
provider: provider.id,
|
||||
model: model.spec.id,
|
||||
resolution: result.resolution,
|
||||
attempts: attempts.length,
|
||||
ms: Date.now() - startedAt,
|
||||
}));
|
||||
logFailure(deps, requestId, req, result.resolution, attempts, status, Date.now() - startedAt, message);
|
||||
return;
|
||||
}
|
||||
// A key fault leaves the pool's other keys worth trying; a provider fault
|
||||
// does not.
|
||||
if (kind === 'provider') benched.add(provider.id);
|
||||
continue;
|
||||
}
|
||||
|
||||
// ---- success -----------------------------------------------------------
|
||||
const headers = routeHeaders({
|
||||
provider: provider.id,
|
||||
model: model.spec.id,
|
||||
resolution: result.resolution,
|
||||
attempts: attempts.length + 1,
|
||||
ms: Date.now() - startedAt,
|
||||
estimated: false,
|
||||
});
|
||||
// Both are derived from caller input; an unescapable byte here would make
|
||||
// writeHead throw and turn a good response into a 500.
|
||||
if (contextNote) headers['x-freeway-context'] = headerSafe(contextNote);
|
||||
if (req.session) headers['x-freeway-session'] = headerSafe(req.session.id);
|
||||
|
||||
if (req.stream && call.response.body !== null) {
|
||||
await streamThrough(deps, res, candidate, call.response, headers, {
|
||||
requestId,
|
||||
req,
|
||||
resolution: result.resolution,
|
||||
attempts,
|
||||
startedAt,
|
||||
promptTokens: estimateTokens(req.promptText),
|
||||
...(entryKey ? { cache: { key: entryKey, scope: semanticScope, queryText, embedding: pendingEmbedding } } : {}),
|
||||
});
|
||||
dl.cancel();
|
||||
return;
|
||||
}
|
||||
|
||||
const bodyJson = await safeJson(call.response);
|
||||
dl.cancel();
|
||||
const usage = usageFromBody(bodyJson, req.promptText, completionText(bodyJson));
|
||||
if (usage.estimated) headers['x-freeway-usage'] = 'estimated';
|
||||
|
||||
const credits = creditsFor(model.spec, provider.spec.limits, usage.promptTokens, usage.completionTokens);
|
||||
store.recordSuccess(provider.id, keyId, call.ms, usage.totalTokens, credits);
|
||||
if (req.caller) deps.guard?.limiter.recordTokens(req.caller.keyId, req.caller.ip, usage.totalTokens);
|
||||
|
||||
if (entryKey && deps.cache && bodyJson) {
|
||||
deps.cache.exact.put(entryKey, JSON.stringify(bodyJson), provider.id, model.spec.id, usage.promptTokens, usage.completionTokens);
|
||||
if (pendingEmbedding && semanticScope && queryText) {
|
||||
deps.cache.semantic?.add(pendingEmbedding, entryKey, semanticScope, queryText);
|
||||
}
|
||||
}
|
||||
if (req.session && deps.context) {
|
||||
recordReply(deps.context.store, req.session.id, completionText(bodyJson), usage.completionTokens);
|
||||
}
|
||||
|
||||
sendJson(res, 200, bodyJson ?? {}, headers);
|
||||
store.addLog({
|
||||
id: requestId,
|
||||
at: startedAt,
|
||||
requestedModel: req.requestedModel,
|
||||
resolution: result.resolution,
|
||||
providerId: provider.id,
|
||||
modelId: model.spec.id,
|
||||
keyId,
|
||||
attempts,
|
||||
status: 200,
|
||||
ms: Date.now() - startedAt,
|
||||
tokensIn: usage.promptTokens,
|
||||
tokensOut: usage.completionTokens,
|
||||
stream: false,
|
||||
cache: null,
|
||||
error: null,
|
||||
preview: { prompt: previewOf(req.promptText), response: previewOf(completionText(bodyJson)) },
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Every candidate was tried and none answered.
|
||||
respondExhausted(deps, res, req, result.blocked, result.resolution, attempts, requestId, startedAt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Answer straight from the cache.
|
||||
*
|
||||
* A streaming client gets the response replayed as SSE, so a hit is
|
||||
* indistinguishable from a live call apart from the `x-freeway-cache` header
|
||||
* and the fact that it cost nothing.
|
||||
*/
|
||||
function serveFromCache(
|
||||
deps: ProxyDeps,
|
||||
res: Res,
|
||||
req: ProxyRequest,
|
||||
entry: CacheEntry,
|
||||
tier: CacheTier,
|
||||
score: number,
|
||||
resolution: string,
|
||||
requestId: string,
|
||||
startedAt: number,
|
||||
): void {
|
||||
const headers = routeHeaders({
|
||||
provider: entry.providerId,
|
||||
model: entry.modelId,
|
||||
resolution,
|
||||
attempts: 0,
|
||||
ms: Date.now() - startedAt,
|
||||
});
|
||||
headers['x-freeway-cache'] = tier;
|
||||
if (tier === 'semantic') headers['x-freeway-cache-score'] = score.toFixed(4);
|
||||
|
||||
if (req.stream) {
|
||||
res.writeHead(200, {
|
||||
'content-type': 'text/event-stream; charset=utf-8',
|
||||
'cache-control': 'no-cache, no-transform',
|
||||
connection: 'keep-alive',
|
||||
...headers,
|
||||
});
|
||||
res.end(replayAsStream(entry.body));
|
||||
} else {
|
||||
let parsed: unknown = null;
|
||||
try {
|
||||
parsed = JSON.parse(entry.body);
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
sendJson(res, 200, parsed ?? {}, headers);
|
||||
}
|
||||
|
||||
// Deliberately not metered: a cache hit spends no provider quota, and
|
||||
// counting it would make the quota bars lie.
|
||||
deps.store.addLog({
|
||||
id: requestId,
|
||||
at: startedAt,
|
||||
requestedModel: req.requestedModel,
|
||||
resolution,
|
||||
providerId: entry.providerId,
|
||||
modelId: entry.modelId,
|
||||
keyId: null,
|
||||
attempts: [],
|
||||
status: 200,
|
||||
ms: Date.now() - startedAt,
|
||||
tokensIn: entry.tokensIn,
|
||||
tokensOut: entry.tokensOut,
|
||||
stream: req.stream,
|
||||
cache: tier,
|
||||
error: null,
|
||||
preview: { prompt: previewOf(req.promptText), response: previewOf(cachedText(entry.body)) },
|
||||
});
|
||||
}
|
||||
|
||||
/** Pull the assistant text out of a stored completion, for the log preview. */
|
||||
function cachedText(body: string): string {
|
||||
try {
|
||||
return completionText(JSON.parse(body));
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Streaming
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface StreamContext {
|
||||
requestId: string;
|
||||
req: ProxyRequest;
|
||||
resolution: string;
|
||||
attempts: AttemptRecord[];
|
||||
startedAt: number;
|
||||
promptTokens: number;
|
||||
/** Cache slot to fill once the stream has finished and its text is known. */
|
||||
cache?: { key: string; scope: string | null; queryText: string; embedding: number[] | null };
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipe the upstream SSE body straight through with backpressure, sniffing the
|
||||
* final chunk for a usage block on the way past. Nothing is buffered: a slow
|
||||
* client slows the upstream read rather than filling memory here.
|
||||
*/
|
||||
async function streamThrough(
|
||||
deps: ProxyDeps,
|
||||
res: Res,
|
||||
candidate: Candidate,
|
||||
response: Response,
|
||||
headers: Record<string, string>,
|
||||
ctx: StreamContext,
|
||||
): Promise<void> {
|
||||
const { store, logger } = deps;
|
||||
const { provider, model, key } = candidate;
|
||||
const keyId = provider.spec.auth.type === 'none' ? null : key.id;
|
||||
|
||||
res.writeHead(200, {
|
||||
'content-type': response.headers.get('content-type') ?? 'text/event-stream; charset=utf-8',
|
||||
'cache-control': 'no-cache, no-transform',
|
||||
connection: 'keep-alive',
|
||||
'x-accel-buffering': 'no',
|
||||
...headers,
|
||||
});
|
||||
|
||||
const sniffer = new UsageSniffer();
|
||||
let failed: string | null = null;
|
||||
|
||||
try {
|
||||
// `response.body` is a web stream; Node's Readable adapter needs the cast.
|
||||
await pipeline(Readable.fromWeb(response.body as WebReadableStream<Uint8Array>), sniffer, res);
|
||||
} catch (err) {
|
||||
failed = err instanceof Error ? err.message : String(err);
|
||||
logger.warn(`stream from ${provider.id}/${model.spec.id} broke: ${failed}`);
|
||||
}
|
||||
|
||||
// Streams only report usage in the very last chunk, which is why this cannot
|
||||
// happen at request time like the non-streaming path.
|
||||
const reported = sniffer.usage;
|
||||
const usage: TokenUsage = reported
|
||||
? {
|
||||
promptTokens: reported.prompt_tokens ?? 0,
|
||||
completionTokens: reported.completion_tokens ?? 0,
|
||||
totalTokens: reported.total_tokens ?? (reported.prompt_tokens ?? 0) + (reported.completion_tokens ?? 0),
|
||||
estimated: false,
|
||||
}
|
||||
: {
|
||||
promptTokens: ctx.promptTokens,
|
||||
completionTokens: estimateTokens(sniffer.text),
|
||||
totalTokens: ctx.promptTokens + estimateTokens(sniffer.text),
|
||||
estimated: true,
|
||||
};
|
||||
|
||||
const credits = creditsFor(model.spec, provider.spec.limits, usage.promptTokens, usage.completionTokens);
|
||||
const ms = Date.now() - ctx.startedAt;
|
||||
|
||||
if (failed === null) {
|
||||
store.recordSuccess(provider.id, keyId, ms, usage.totalTokens, credits);
|
||||
if (ctx.req.caller) deps.guard?.limiter.recordTokens(ctx.req.caller.keyId, ctx.req.caller.ip, usage.totalTokens);
|
||||
// A stream's text is only complete once the pipe finishes, which is why the
|
||||
// session write happens here rather than at commit time.
|
||||
if (ctx.req.session && deps.context) recordReply(deps.context.store, ctx.req.session.id, sniffer.text, usage.completionTokens);
|
||||
} else {
|
||||
store.recordFailure(provider.id, keyId, { status: null, message: failed });
|
||||
}
|
||||
|
||||
// A stream can only be cached once it has finished, because its text does not
|
||||
// exist until then. Reassembled into the non-streaming shape so a later hit
|
||||
// can be replayed either way.
|
||||
if (failed === null && ctx.cache && deps.cache && sniffer.text) {
|
||||
const assembled = JSON.stringify({
|
||||
id: `chatcmpl-${ctx.requestId}`,
|
||||
object: 'chat.completion',
|
||||
created: Math.floor(ctx.startedAt / 1000),
|
||||
model: model.spec.id,
|
||||
choices: [{ index: 0, message: { role: 'assistant', content: sniffer.text }, finish_reason: 'stop' }],
|
||||
usage: { prompt_tokens: usage.promptTokens, completion_tokens: usage.completionTokens, total_tokens: usage.totalTokens },
|
||||
});
|
||||
deps.cache.exact.put(ctx.cache.key, assembled, provider.id, model.spec.id, usage.promptTokens, usage.completionTokens);
|
||||
if (ctx.cache.embedding && ctx.cache.scope && ctx.cache.queryText) {
|
||||
deps.cache.semantic?.add(ctx.cache.embedding, ctx.cache.key, ctx.cache.scope, ctx.cache.queryText);
|
||||
}
|
||||
}
|
||||
|
||||
store.addLog({
|
||||
id: ctx.requestId,
|
||||
at: ctx.startedAt,
|
||||
requestedModel: ctx.req.requestedModel,
|
||||
resolution: ctx.resolution,
|
||||
providerId: provider.id,
|
||||
modelId: model.spec.id,
|
||||
keyId,
|
||||
attempts: ctx.attempts,
|
||||
status: failed === null ? 200 : 499,
|
||||
ms,
|
||||
tokensIn: usage.promptTokens,
|
||||
tokensOut: usage.completionTokens,
|
||||
stream: true,
|
||||
cache: null,
|
||||
error: failed,
|
||||
preview: { prompt: previewOf(ctx.req.promptText), response: previewOf(sniffer.text) },
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Failure responses
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function respondAllBlocked(
|
||||
deps: ProxyDeps,
|
||||
res: Res,
|
||||
req: ProxyRequest,
|
||||
blocked: Blocked[],
|
||||
resolution: string,
|
||||
requestId: string,
|
||||
startedAt: number,
|
||||
): void {
|
||||
const soonest = shortestRetry(blocked);
|
||||
const headers = routeHeaders({ resolution, attempts: 0, ms: Date.now() - startedAt });
|
||||
if (soonest !== null) headers['retry-after'] = String(Math.max(1, Math.ceil(soonest / 1000)));
|
||||
|
||||
sendError(
|
||||
res,
|
||||
429,
|
||||
{
|
||||
message: `no provider currently available for model "${req.requestedModel}" — ${blocked.length} candidate${blocked.length === 1 ? '' : 's'} blocked`,
|
||||
type: 'rate_limit_error',
|
||||
code: 'all_providers_blocked',
|
||||
},
|
||||
{
|
||||
requested: req.requestedModel,
|
||||
resolution,
|
||||
retryAfterMs: soonest,
|
||||
blocked: blocked.map((b) => ({ provider: b.providerId, model: b.modelId, reason: b.reason, retryAfterMs: b.retryAfterMs })),
|
||||
},
|
||||
headers,
|
||||
);
|
||||
|
||||
logFailure(deps, requestId, req, resolution, [], 429, Date.now() - startedAt, 'all providers blocked');
|
||||
}
|
||||
|
||||
function respondExhausted(
|
||||
deps: ProxyDeps,
|
||||
res: Res,
|
||||
req: ProxyRequest,
|
||||
blocked: Blocked[],
|
||||
resolution: string,
|
||||
attempts: AttemptRecord[],
|
||||
requestId: string,
|
||||
startedAt: number,
|
||||
): void {
|
||||
const lastStatus = attempts.length > 0 ? attempts[attempts.length - 1]?.status : null;
|
||||
const status = lastStatus === 429 ? 429 : 502;
|
||||
const headers = routeHeaders({ resolution, attempts: attempts.length, ms: Date.now() - startedAt });
|
||||
|
||||
sendError(
|
||||
res,
|
||||
status,
|
||||
{
|
||||
message: `every provider failed for model "${req.requestedModel}" after ${attempts.length} attempt${attempts.length === 1 ? '' : 's'}`,
|
||||
type: status === 429 ? 'rate_limit_error' : 'api_error',
|
||||
code: 'all_attempts_failed',
|
||||
},
|
||||
{
|
||||
requested: req.requestedModel,
|
||||
resolution,
|
||||
attempts: attempts.map((a) => ({ provider: a.providerId, model: a.modelId, status: a.status, ms: a.ms, error: a.error })),
|
||||
blocked: blocked.map((b) => ({ provider: b.providerId, model: b.modelId, reason: b.reason, retryAfterMs: b.retryAfterMs })),
|
||||
},
|
||||
headers,
|
||||
);
|
||||
|
||||
logFailure(deps, requestId, req, resolution, attempts, status, Date.now() - startedAt, 'all attempts failed');
|
||||
}
|
||||
|
||||
function logFailure(
|
||||
deps: ProxyDeps,
|
||||
requestId: string,
|
||||
req: ProxyRequest,
|
||||
resolution: string,
|
||||
attempts: AttemptRecord[],
|
||||
status: number,
|
||||
ms: number,
|
||||
error: string,
|
||||
): void {
|
||||
const last = attempts[attempts.length - 1];
|
||||
deps.store.addLog({
|
||||
id: requestId,
|
||||
at: Date.now() - ms,
|
||||
requestedModel: req.requestedModel,
|
||||
resolution,
|
||||
providerId: last?.providerId ?? null,
|
||||
modelId: last?.modelId ?? null,
|
||||
keyId: last?.keyId ?? null,
|
||||
attempts,
|
||||
status,
|
||||
ms,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
stream: req.stream,
|
||||
cache: null,
|
||||
error,
|
||||
preview: { prompt: previewOf(req.promptText), response: '' },
|
||||
});
|
||||
}
|
||||
|
||||
function shortestRetry(blocked: Blocked[]): number | null {
|
||||
let soonest: number | null = null;
|
||||
for (const b of blocked) {
|
||||
if (b.retryAfterMs === null) continue;
|
||||
soonest = soonest === null ? b.retryAfterMs : Math.min(soonest, b.retryAfterMs);
|
||||
}
|
||||
return soonest;
|
||||
}
|
||||
|
||||
interface RouteHeaderInput {
|
||||
provider?: string;
|
||||
model?: string;
|
||||
resolution: string;
|
||||
attempts: number;
|
||||
ms: number;
|
||||
estimated?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip anything a header value cannot legally carry.
|
||||
*
|
||||
* Resolution strings are built from user-supplied model names and provider ids,
|
||||
* so a stray non-latin-1 byte would make `writeHead` throw and turn a perfectly
|
||||
* good response into a 500. Sanitising at the boundary means no future caller
|
||||
* has to remember this.
|
||||
*/
|
||||
function headerSafe(value: string): string {
|
||||
// eslint-disable-next-line no-control-regex -- control chars are exactly what must go
|
||||
return value.replace(/[^\x20-\x7E]/g, '?').slice(0, 512);
|
||||
}
|
||||
|
||||
/** Every reply carries these, so a curl tells you what happened without the dashboard. */
|
||||
function routeHeaders(input: RouteHeaderInput): Record<string, string> {
|
||||
const headers: Record<string, string> = {
|
||||
'x-freeway-route': headerSafe(input.resolution),
|
||||
'x-freeway-attempts': String(input.attempts),
|
||||
'x-freeway-ms': String(input.ms),
|
||||
};
|
||||
if (input.provider) headers['x-freeway-provider'] = headerSafe(input.provider);
|
||||
if (input.model) headers['x-freeway-model'] = headerSafe(input.model);
|
||||
return headers;
|
||||
}
|
||||
|
||||
async function safeJson(response: Response): Promise<Record<string, unknown> | null> {
|
||||
try {
|
||||
const text = await response.text();
|
||||
if (!text) return null;
|
||||
const parsed: unknown = JSON.parse(text);
|
||||
return typeof parsed === 'object' && parsed !== null ? (parsed as Record<string, unknown>) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function extractMessage(body: Record<string, unknown> | null): string | null {
|
||||
const error = body?.['error'];
|
||||
if (typeof error === 'string') return error;
|
||||
if (typeof error === 'object' && error !== null) {
|
||||
const message = (error as { message?: unknown }).message;
|
||||
if (typeof message === 'string') return message;
|
||||
}
|
||||
const message = body?.['message'];
|
||||
return typeof message === 'string' ? message : null;
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Session hydration: turn `{ session, messages: [newMessage] }` into the full
|
||||
* conversation, and persist the reply afterwards.
|
||||
*
|
||||
* The `session` field is a Freeway extension and is stripped before anything is
|
||||
* forwarded upstream — no provider knows about it.
|
||||
*/
|
||||
|
||||
import { ContextStore, Tokenizer, textOf, type RefitMessage } from '@freeway/core';
|
||||
|
||||
export interface SessionContext {
|
||||
id: string;
|
||||
/** Sequence number of the last message that existed before this request. */
|
||||
baseSeq: number;
|
||||
/** How many stored turns were rehydrated. */
|
||||
restored: number;
|
||||
}
|
||||
|
||||
export interface HydrateResult {
|
||||
messages: RefitMessage[];
|
||||
session: SessionContext | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge stored history with what the client just sent.
|
||||
*
|
||||
* Clients are allowed to send the whole array as usual — in that case the
|
||||
* stored history is ignored for anything they re-sent, so a session id is
|
||||
* additive rather than a different protocol.
|
||||
*/
|
||||
export class SessionForbiddenError extends Error {
|
||||
constructor(sessionId: string) {
|
||||
super(`session "${sessionId}" belongs to a different key`);
|
||||
this.name = 'SessionForbiddenError';
|
||||
}
|
||||
}
|
||||
|
||||
export function hydrate(
|
||||
store: ContextStore,
|
||||
sessionId: string | null,
|
||||
incoming: RefitMessage[],
|
||||
tokenizer: Tokenizer,
|
||||
model?: string,
|
||||
owner: string | null = null,
|
||||
): HydrateResult {
|
||||
if (!sessionId) return { messages: incoming, session: null };
|
||||
|
||||
// Session ids are caller-chosen strings. Without this check, "proj-42" is
|
||||
// readable by anyone who guesses it.
|
||||
if (!store.canAccess(sessionId, owner)) throw new SessionForbiddenError(sessionId);
|
||||
|
||||
const stored = store.messages(sessionId);
|
||||
const baseSeq = stored.length > 0 ? (stored[stored.length - 1]?.seq ?? -1) : -1;
|
||||
|
||||
if (stored.length === 0) {
|
||||
// First turn of a new session: everything the client sent is the history.
|
||||
store.append(
|
||||
sessionId,
|
||||
incoming.map((m) => ({
|
||||
role: m.role,
|
||||
content: textOf(m.content),
|
||||
extra: extraOf(m),
|
||||
tokens: tokenizer.estimate(textOf(m.content), model),
|
||||
})),
|
||||
Date.now(),
|
||||
owner,
|
||||
);
|
||||
return { messages: incoming, session: { id: sessionId, baseSeq: -1, restored: 0 } };
|
||||
}
|
||||
|
||||
const history: RefitMessage[] = stored.map((m) => {
|
||||
const extra = m.extra ? (JSON.parse(m.extra) as Record<string, unknown>) : {};
|
||||
return { role: m.role, content: m.content, ...extra };
|
||||
});
|
||||
|
||||
// Only messages the client did not already replay get appended, so a client
|
||||
// that keeps its own copy and one that relies on the server both work.
|
||||
const known = new Set(history.map(fingerprint));
|
||||
const fresh = incoming.filter((m) => !known.has(fingerprint(m)));
|
||||
|
||||
if (fresh.length > 0) {
|
||||
store.append(
|
||||
sessionId,
|
||||
fresh.map((m) => ({
|
||||
role: m.role,
|
||||
content: textOf(m.content),
|
||||
extra: extraOf(m),
|
||||
tokens: tokenizer.estimate(textOf(m.content), model),
|
||||
})),
|
||||
Date.now(),
|
||||
owner,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
messages: [...history, ...fresh],
|
||||
session: { id: sessionId, baseSeq, restored: history.length },
|
||||
};
|
||||
}
|
||||
|
||||
/** Persist the assistant's reply so the next turn sees it. */
|
||||
export function recordReply(store: ContextStore, sessionId: string, content: string, tokens: number): void {
|
||||
if (!content) return;
|
||||
store.append(sessionId, [{ role: 'assistant', content, extra: null, tokens }]);
|
||||
}
|
||||
|
||||
function extraOf(m: RefitMessage): string | null {
|
||||
const extra: Record<string, unknown> = {};
|
||||
for (const [k, v] of Object.entries(m)) {
|
||||
if (k === 'role' || k === 'content') continue;
|
||||
extra[k] = v;
|
||||
}
|
||||
return Object.keys(extra).length > 0 ? JSON.stringify(extra) : null;
|
||||
}
|
||||
|
||||
/** Cheap identity for dedupe. Content is bounded so a huge message stays cheap. */
|
||||
function fingerprint(m: RefitMessage): string {
|
||||
return `${m.role}:${textOf(m.content).slice(0, 200)}`;
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* A pass-through Transform that watches an SSE stream go by and picks the
|
||||
* `usage` block out of it.
|
||||
*
|
||||
* Providers only report token usage in the final chunk of a stream, so this is
|
||||
* the only place the real numbers can be captured. Bytes are forwarded
|
||||
* unmodified — the client receives exactly what the provider sent.
|
||||
*/
|
||||
|
||||
import { Transform, type TransformCallback } from 'node:stream';
|
||||
|
||||
export interface StreamUsage {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
total_tokens?: number;
|
||||
}
|
||||
|
||||
/** Guard against a provider that never sends a newline filling memory. */
|
||||
const MAX_PARTIAL_LINE = 256 * 1024;
|
||||
|
||||
export class UsageSniffer extends Transform {
|
||||
usage: StreamUsage | null = null;
|
||||
/** Accumulated assistant text, for estimating when no usage is reported. */
|
||||
text = '';
|
||||
|
||||
private partial = '';
|
||||
|
||||
override _transform(chunk: Buffer | string, _encoding: BufferEncoding, callback: TransformCallback): void {
|
||||
// Forward first and unchanged: inspection must never alter what the client
|
||||
// receives, and must never delay it either.
|
||||
this.push(chunk);
|
||||
|
||||
this.partial += typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
||||
if (this.partial.length > MAX_PARTIAL_LINE) {
|
||||
// Not a real SSE stream, or a pathologically long line. Stop accumulating
|
||||
// rather than growing without bound; pass-through is unaffected.
|
||||
this.partial = '';
|
||||
callback();
|
||||
return;
|
||||
}
|
||||
|
||||
const lines = this.partial.split('\n');
|
||||
this.partial = lines.pop() ?? '';
|
||||
for (const line of lines) this.inspect(line);
|
||||
callback();
|
||||
}
|
||||
|
||||
override _flush(callback: TransformCallback): void {
|
||||
if (this.partial) this.inspect(this.partial);
|
||||
this.partial = '';
|
||||
callback();
|
||||
}
|
||||
|
||||
private inspect(line: string): void {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed.startsWith('data:')) return;
|
||||
const payload = trimmed.slice(5).trim();
|
||||
if (!payload || payload === '[DONE]') return;
|
||||
|
||||
// Parsing every chunk would be wasteful; only chunks that could carry the
|
||||
// fields we want are worth the JSON.parse.
|
||||
const hasUsage = payload.includes('"usage"');
|
||||
const hasContent = payload.includes('"content"');
|
||||
if (!hasUsage && !hasContent) return;
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(payload);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (typeof parsed !== 'object' || parsed === null) return;
|
||||
|
||||
const usage = (parsed as { usage?: unknown }).usage;
|
||||
if (typeof usage === 'object' && usage !== null) {
|
||||
const u = usage as Record<string, unknown>;
|
||||
this.usage = {
|
||||
...(typeof u['prompt_tokens'] === 'number' ? { prompt_tokens: u['prompt_tokens'] } : {}),
|
||||
...(typeof u['completion_tokens'] === 'number' ? { completion_tokens: u['completion_tokens'] } : {}),
|
||||
...(typeof u['total_tokens'] === 'number' ? { total_tokens: u['total_tokens'] } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
const choices = (parsed as { choices?: unknown }).choices;
|
||||
if (Array.isArray(choices)) {
|
||||
for (const choice of choices) {
|
||||
const delta = (choice as { delta?: { content?: unknown } } | null)?.delta;
|
||||
if (typeof delta?.content === 'string') this.text += delta.content;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
/**
|
||||
* Calling a real provider: auth, header assembly, param stripping, timeouts,
|
||||
* and pulling usage back out of the response.
|
||||
*
|
||||
* Everything here is driven by the provider's JSON spec. Nothing in this file
|
||||
* knows the name of a single provider, which is what keeps "add a provider by
|
||||
* writing JSON" true.
|
||||
*/
|
||||
|
||||
import type { Candidate, Cap } from '@freeway/core';
|
||||
import { joinUrl } from '@freeway/core';
|
||||
|
||||
export interface UpstreamCall {
|
||||
candidate: Candidate;
|
||||
/** Path relative to the provider's baseUrl, e.g. `/chat/completions`. */
|
||||
path: string;
|
||||
body: unknown;
|
||||
/**
|
||||
* Owned by the caller, not by this function. A streamed response is still
|
||||
* arriving long after `fetch` resolves, so the deadline has to outlive the
|
||||
* call itself — clearing a timer here would leave a hung stream immortal.
|
||||
*/
|
||||
signal: AbortSignal;
|
||||
method?: string;
|
||||
}
|
||||
|
||||
export interface Deadline {
|
||||
signal: AbortSignal;
|
||||
cancel: () => void;
|
||||
timedOut: () => boolean;
|
||||
}
|
||||
|
||||
export function deadline(ms: number): Deadline {
|
||||
const controller = new AbortController();
|
||||
let fired = false;
|
||||
const timer = setTimeout(() => {
|
||||
fired = true;
|
||||
controller.abort();
|
||||
}, ms);
|
||||
return {
|
||||
signal: controller.signal,
|
||||
cancel: () => clearTimeout(timer),
|
||||
timedOut: () => fired,
|
||||
};
|
||||
}
|
||||
|
||||
export interface UpstreamResult {
|
||||
ok: boolean;
|
||||
status: number | null;
|
||||
response: Response | null;
|
||||
/** Set when the call never produced an HTTP response (network, DNS, timeout). */
|
||||
transportError: string | null;
|
||||
timedOut: boolean;
|
||||
ms: number;
|
||||
}
|
||||
|
||||
export interface TokenUsage {
|
||||
promptTokens: number;
|
||||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
/** True when the numbers are our own estimate rather than the provider's. */
|
||||
estimated: boolean;
|
||||
}
|
||||
|
||||
/** Build the URL and headers for one candidate, applying its auth scheme. */
|
||||
export function buildRequest(candidate: Candidate, path: string): { url: string; headers: Record<string, string> } {
|
||||
const { provider, key } = candidate;
|
||||
const auth = provider.spec.auth;
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'content-type': 'application/json',
|
||||
accept: 'application/json',
|
||||
...provider.headers,
|
||||
};
|
||||
|
||||
let url = joinUrl(provider.baseUrl, path);
|
||||
|
||||
switch (auth.type) {
|
||||
case 'bearer':
|
||||
headers['authorization'] = `${auth.prefix ?? 'Bearer '}${key.value}`;
|
||||
break;
|
||||
case 'header':
|
||||
if (auth.header) headers[auth.header.toLowerCase()] = `${auth.prefix ?? ''}${key.value}`;
|
||||
break;
|
||||
case 'query':
|
||||
if (auth.query) {
|
||||
const u = new URL(url);
|
||||
u.searchParams.set(auth.query, key.value);
|
||||
url = u.toString();
|
||||
}
|
||||
break;
|
||||
case 'none':
|
||||
break;
|
||||
}
|
||||
|
||||
return { url, headers };
|
||||
}
|
||||
|
||||
/** Remove body params this provider rejects. Returns a copy; the input is untouched. */
|
||||
export function stripDropParams(body: unknown, dropParams: string[]): unknown {
|
||||
if (dropParams.length === 0 || typeof body !== 'object' || body === null || Array.isArray(body)) return body;
|
||||
const out: Record<string, unknown> = { ...(body as Record<string, unknown>) };
|
||||
for (const p of dropParams) delete out[p];
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function callUpstream(call: UpstreamCall): Promise<UpstreamResult> {
|
||||
const { candidate, path, signal } = call;
|
||||
const { url, headers } = buildRequest(candidate, path);
|
||||
const body = stripDropParams(call.body, candidate.provider.spec.dropParams);
|
||||
const started = Date.now();
|
||||
|
||||
const method = call.method ?? 'POST';
|
||||
const init: RequestInit = { method, headers, signal };
|
||||
if (method !== 'GET' && method !== 'HEAD') init.body = JSON.stringify(body);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, init);
|
||||
return {
|
||||
ok: response.ok,
|
||||
status: response.status,
|
||||
response,
|
||||
transportError: null,
|
||||
timedOut: false,
|
||||
ms: Date.now() - started,
|
||||
};
|
||||
} catch (err) {
|
||||
const timedOut = signal.aborted;
|
||||
return {
|
||||
ok: false,
|
||||
status: null,
|
||||
response: null,
|
||||
transportError: timedOut ? 'upstream timed out' : errMsg(err),
|
||||
timedOut,
|
||||
ms: Date.now() - started,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Honour an upstream `Retry-After`, which may be seconds or an HTTP date.
|
||||
* A provider telling us when to come back is always better than our guess.
|
||||
*/
|
||||
export function retryAfterMs(response: Response | null): number | null {
|
||||
const raw = response?.headers.get('retry-after');
|
||||
if (!raw) return null;
|
||||
const seconds = Number(raw);
|
||||
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000;
|
||||
const date = Date.parse(raw);
|
||||
if (Number.isFinite(date)) return Math.max(0, date - Date.now());
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Rough token count. Only used when a provider reports nothing. */
|
||||
export function estimateTokens(text: string): number {
|
||||
return Math.max(1, Math.ceil(text.length / 4));
|
||||
}
|
||||
|
||||
export function textOfMessages(messages: unknown): string {
|
||||
if (!Array.isArray(messages)) return '';
|
||||
const parts: string[] = [];
|
||||
for (const m of messages) {
|
||||
if (typeof m !== 'object' || m === null) continue;
|
||||
const content = (m as { content?: unknown }).content;
|
||||
if (typeof content === 'string') parts.push(content);
|
||||
else if (Array.isArray(content)) {
|
||||
for (const p of content) {
|
||||
if (typeof p === 'object' && p !== null && typeof (p as { text?: unknown }).text === 'string') {
|
||||
parts.push((p as { text: string }).text);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull usage out of a non-streaming response body, falling back to an estimate.
|
||||
*
|
||||
* Degrading to `chars/4` matters: a provider that reports nothing would
|
||||
* otherwise leave every quota counter reading zero, and the gateway would
|
||||
* cheerfully burn a monthly allowance believing it had spent nothing.
|
||||
*/
|
||||
export function usageFromBody(body: unknown, promptText: string, completionText: string): TokenUsage {
|
||||
const usage = (body as { usage?: Record<string, unknown> } | null)?.usage;
|
||||
const prompt = numberAt(usage, 'prompt_tokens');
|
||||
const completion = numberAt(usage, 'completion_tokens');
|
||||
const total = numberAt(usage, 'total_tokens');
|
||||
|
||||
if (prompt !== null || completion !== null || total !== null) {
|
||||
const p = prompt ?? 0;
|
||||
const c = completion ?? 0;
|
||||
return { promptTokens: p, completionTokens: c, totalTokens: total ?? p + c, estimated: false };
|
||||
}
|
||||
|
||||
const p = estimateTokens(promptText);
|
||||
const c = estimateTokens(completionText);
|
||||
return { promptTokens: p, completionTokens: c, totalTokens: p + c, estimated: true };
|
||||
}
|
||||
|
||||
export function completionText(body: unknown): string {
|
||||
const choices = (body as { choices?: unknown } | null)?.choices;
|
||||
if (!Array.isArray(choices)) return '';
|
||||
const parts: string[] = [];
|
||||
for (const c of choices) {
|
||||
const message = (c as { message?: { content?: unknown } } | null)?.message;
|
||||
if (typeof message?.content === 'string') parts.push(message.content);
|
||||
}
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
/** Capabilities the request body implies. Drives the router's filtering. */
|
||||
export function capsFromBody(body: Record<string, unknown>): Cap[] {
|
||||
const caps: Cap[] = [];
|
||||
const tools = body['tools'];
|
||||
if (Array.isArray(tools) && tools.length > 0) caps.push('tools');
|
||||
if (body['functions'] !== undefined && !caps.includes('tools')) caps.push('tools');
|
||||
|
||||
const format = body['response_format'];
|
||||
if (typeof format === 'object' && format !== null) {
|
||||
const type = (format as { type?: unknown }).type;
|
||||
if (type === 'json_object' || type === 'json_schema') caps.push('json');
|
||||
}
|
||||
|
||||
if (hasImagePart(body['messages'])) caps.push('vision');
|
||||
return caps;
|
||||
}
|
||||
|
||||
function hasImagePart(messages: unknown): boolean {
|
||||
if (!Array.isArray(messages)) return false;
|
||||
for (const m of messages) {
|
||||
const content = (m as { content?: unknown } | null)?.content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
for (const part of content) {
|
||||
const type = (part as { type?: unknown } | null)?.type;
|
||||
if (type === 'image_url' || type === 'image' || type === 'input_image') return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function numberAt(obj: Record<string, unknown> | undefined, key: string): number | null {
|
||||
const v = obj?.[key];
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : null;
|
||||
}
|
||||
|
||||
function errMsg(err: unknown): string {
|
||||
if (err instanceof Error) {
|
||||
const cause = (err as { cause?: unknown }).cause;
|
||||
if (cause instanceof Error && cause.message) return `${err.message}: ${cause.message}`;
|
||||
return err.message;
|
||||
}
|
||||
return String(err);
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* Cache behaviour through the whole stack: a hit must cost no upstream call and
|
||||
* no quota, and must be indistinguishable from a live answer to the client.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { ExactCache, Registry, Router, SemanticCache, Store, parseConfig, silentLogger, type CacheMode, type FreewayConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from '../src/server.ts';
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock {
|
||||
url: string;
|
||||
callsTo(path: string): unknown[];
|
||||
close(): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
base: string;
|
||||
upstream: Mock;
|
||||
cache: ExactCache;
|
||||
semantic: SemanticCache;
|
||||
store: Store;
|
||||
}
|
||||
|
||||
async function harness(mode: CacheMode = 'safe', semanticOn = false): Promise<Harness> {
|
||||
const upstream = (await createMockUpstream({ name: 'up' })) as Mock;
|
||||
cleanups.push(() => upstream.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-cache-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
writeFileSync(join(dir, 'up.json'), JSON.stringify({
|
||||
id: 'up', priority: 10, baseUrl: upstream.url,
|
||||
auth: { type: 'bearer', envKeys: ['UP_KEY'] }, limits: { rpm: 1000 },
|
||||
models: [
|
||||
{ id: 'up-chat', alias: ['fast'], context: 128000, caps: ['chat'] },
|
||||
{ id: 'up-embed', alias: ['embed'], context: 8000, caps: ['embed'] },
|
||||
],
|
||||
}));
|
||||
|
||||
const registry = new Registry({ dir, env: { UP_KEY: 'uk-000000000001' }, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {}) as { config: FreewayConfig };
|
||||
config.cache.mode = mode;
|
||||
config.cache.semantic.enabled = semanticOn;
|
||||
const router = new Router(registry, store, config);
|
||||
|
||||
const cache = new ExactCache({ file: ':memory:', mode, ttlMs: 3_600_000, maxEntries: 100 });
|
||||
cleanups.push(() => cache.close());
|
||||
const semantic = new SemanticCache({ enabled: semanticOn, threshold: 0.92, maxEntries: 100, dims: 8 });
|
||||
|
||||
const server: Server = createGateway({
|
||||
registry, store, router, config, logger: silentLogger, cache, semanticCache: semantic,
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
return { base: `http://127.0.0.1:${port}`, upstream, cache, semantic, store };
|
||||
}
|
||||
|
||||
async function chat(base: string, body: unknown): Promise<Response> {
|
||||
return fetch(`${base}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
const ask = (content: string, over: Record<string, unknown> = {}) => ({
|
||||
model: 'fast',
|
||||
temperature: 0,
|
||||
messages: [{ role: 'user', content }],
|
||||
...over,
|
||||
});
|
||||
|
||||
describe('cache — exact hits', () => {
|
||||
it('serves the second identical request without touching the upstream', async () => {
|
||||
const h = await harness();
|
||||
|
||||
const first = await chat(h.base, ask('what is the capital of France?'));
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(first.headers.get('x-freeway-cache'), null, 'the first call is a miss');
|
||||
const firstBody = await first.text();
|
||||
|
||||
const second = await chat(h.base, ask('what is the capital of France?'));
|
||||
assert.equal(second.status, 200);
|
||||
assert.equal(second.headers.get('x-freeway-cache'), 'exact');
|
||||
assert.equal(await second.text(), firstBody, 'byte-identical to the original answer');
|
||||
|
||||
assert.equal(h.upstream.callsTo('/chat/completions').length, 1, 'only one upstream call was ever made');
|
||||
});
|
||||
|
||||
it('spends no quota on a hit', async () => {
|
||||
// Metering a cache hit would make the quota bars lie about what is left.
|
||||
const h = await harness();
|
||||
await chat(h.base, ask('question'));
|
||||
const after = h.store.provider('up').meter.used('rpm');
|
||||
|
||||
await chat(h.base, ask('question'));
|
||||
assert.equal(h.store.provider('up').meter.used('rpm'), after, 'the hit did not count against the provider');
|
||||
});
|
||||
|
||||
it('logs a hit distinctly', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, ask('question'));
|
||||
await chat(h.base, ask('question'));
|
||||
|
||||
const logs = h.store.logs(2);
|
||||
assert.equal(logs[0]?.cache, 'exact');
|
||||
assert.equal(logs[1]?.cache, null);
|
||||
assert.equal(logs[0]?.status, 200);
|
||||
});
|
||||
|
||||
it('does not cache a creative call in safe mode', async () => {
|
||||
const h = await harness('safe');
|
||||
await chat(h.base, ask('write me a poem', { temperature: 0.9 }));
|
||||
const second = await chat(h.base, ask('write me a poem', { temperature: 0.9 }));
|
||||
|
||||
assert.equal(second.headers.get('x-freeway-cache'), null);
|
||||
assert.equal(h.upstream.callsTo('/chat/completions').length, 2, 'both calls went upstream');
|
||||
});
|
||||
|
||||
it('caches a creative call in aggressive mode', async () => {
|
||||
const h = await harness('aggressive');
|
||||
await chat(h.base, ask('write me a poem', { temperature: 0.9 }));
|
||||
const second = await chat(h.base, ask('write me a poem', { temperature: 0.9 }));
|
||||
assert.equal(second.headers.get('x-freeway-cache'), 'exact');
|
||||
});
|
||||
|
||||
it('is inert when off', async () => {
|
||||
const h = await harness('off');
|
||||
await chat(h.base, ask('question'));
|
||||
const second = await chat(h.base, ask('question'));
|
||||
assert.equal(second.headers.get('x-freeway-cache'), null);
|
||||
assert.equal(h.upstream.callsTo('/chat/completions').length, 2);
|
||||
});
|
||||
|
||||
it('treats a different question as a different request', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, ask('question one'));
|
||||
const second = await chat(h.base, ask('question two'));
|
||||
assert.equal(second.headers.get('x-freeway-cache'), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cache — streaming replay', () => {
|
||||
it('replays a cached answer as SSE the client cannot distinguish', async () => {
|
||||
const h = await harness('aggressive');
|
||||
|
||||
const live = await chat(h.base, ask('stream me', { stream: true }));
|
||||
assert.match(live.headers.get('content-type') ?? '', /text\/event-stream/);
|
||||
const liveText = await live.text();
|
||||
|
||||
const cached = await chat(h.base, ask('stream me', { stream: true }));
|
||||
assert.equal(cached.headers.get('x-freeway-cache'), 'exact');
|
||||
assert.match(cached.headers.get('content-type') ?? '', /text\/event-stream/);
|
||||
|
||||
const cachedText = await cached.text();
|
||||
assert.ok(cachedText.trimEnd().endsWith('data: [DONE]'));
|
||||
|
||||
const contentOf = (sse: string): string =>
|
||||
[...sse.matchAll(/^data: (.+)$/gm)]
|
||||
.map((m) => m[1])
|
||||
.filter((p): p is string => p !== undefined && p !== '[DONE]')
|
||||
.map((p) => JSON.parse(p) as { choices?: { delta?: { content?: string } }[] })
|
||||
.flatMap((o) => o.choices ?? [])
|
||||
.map((c) => c.delta?.content ?? '')
|
||||
.join('');
|
||||
|
||||
assert.equal(contentOf(cachedText), contentOf(liveText), 'the replayed text matches the original exactly');
|
||||
});
|
||||
});
|
||||
|
||||
describe('cache — semantic tier', () => {
|
||||
it('hits on a differently-worded question above the threshold', async () => {
|
||||
// The mock's embeddings are deterministic from the text, so similar text
|
||||
// gives similar vectors — the same property a real embedder has.
|
||||
const h = await harness('safe', true);
|
||||
|
||||
await chat(h.base, ask('what is the capital of France'));
|
||||
const before = h.upstream.callsTo('/chat/completions').length;
|
||||
|
||||
const reworded = await chat(h.base, ask('what is the capital of France?'));
|
||||
assert.equal(reworded.status, 200);
|
||||
|
||||
if (reworded.headers.get('x-freeway-cache') === 'semantic') {
|
||||
const score = Number(reworded.headers.get('x-freeway-cache-score'));
|
||||
assert.ok(score >= 0.92, `a hit must clear the threshold, got ${score}`);
|
||||
assert.equal(h.upstream.callsTo('/chat/completions').length, before, 'no chat call was made');
|
||||
}
|
||||
});
|
||||
|
||||
it('never serves across a different system prompt', async () => {
|
||||
// The same question can have a different correct answer under a different
|
||||
// system prompt, so entries must be scoped.
|
||||
const h = await harness('safe', true);
|
||||
|
||||
await chat(h.base, { ...ask('who are you'), messages: [{ role: 'system', content: 'You are a pirate.' }, { role: 'user', content: 'who are you' }] });
|
||||
const other = await chat(h.base, { ...ask('who are you'), messages: [{ role: 'system', content: 'You are a lawyer.' }, { role: 'user', content: 'who are you' }] });
|
||||
|
||||
assert.equal(other.headers.get('x-freeway-cache'), null, 'a different system prompt must not reuse the answer');
|
||||
});
|
||||
|
||||
it('does not consult the semantic tier when disabled', async () => {
|
||||
const h = await harness('safe', false);
|
||||
await chat(h.base, ask('a question'));
|
||||
await chat(h.base, ask('a slightly different question'));
|
||||
assert.equal(h.upstream.callsTo('/embeddings').length, 0, 'no embedding quota was spent');
|
||||
});
|
||||
});
|
||||
|
||||
describe('cache — API', () => {
|
||||
it('reports stats and can be cleared', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, ask('question'));
|
||||
await chat(h.base, ask('question'));
|
||||
|
||||
const stats = (await (await fetch(`${h.base}/api/cache`)).json()) as {
|
||||
enabled: boolean; mode: string; exact: { hits: number; entries: number; tokensSaved: number };
|
||||
};
|
||||
assert.equal(stats.enabled, true);
|
||||
assert.equal(stats.mode, 'safe');
|
||||
assert.equal(stats.exact.hits, 1);
|
||||
assert.equal(stats.exact.entries, 1);
|
||||
assert.ok(stats.exact.tokensSaved > 0);
|
||||
|
||||
assert.equal((await fetch(`${h.base}/api/cache`, { method: 'DELETE' })).status, 200);
|
||||
const after = (await (await fetch(`${h.base}/api/cache`)).json()) as { exact: { entries: number } };
|
||||
assert.equal(after.exact.entries, 0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,665 @@
|
||||
/**
|
||||
* End-to-end tests against mock upstreams.
|
||||
*
|
||||
* Everything here runs a real HTTP gateway against real HTTP fakes — no
|
||||
* stubbing of fetch, no injected doubles — so what is verified is what actually
|
||||
* happens on the wire, and it costs no free-tier quota to run.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, beforeEach, describe, it } from 'node:test';
|
||||
|
||||
import { Registry, Router, Store, parseConfig, silentLogger, type FreewayConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from '../src/server.ts';
|
||||
// eslint-disable-next-line -- plain JS harness, deliberately not TypeScript so it can be run standalone
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock {
|
||||
url: string;
|
||||
name: string;
|
||||
calls: { path: string; body: unknown; key: string | null }[];
|
||||
script(steps: unknown[]): void;
|
||||
setKeys(k: Record<string, unknown>): void;
|
||||
callsTo(path: string): { path: string; body: unknown; key: string | null }[];
|
||||
reset(): void;
|
||||
close(): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
base: string;
|
||||
alpha: Mock;
|
||||
beta: Mock;
|
||||
store: Store;
|
||||
registry: Registry;
|
||||
config: FreewayConfig;
|
||||
}
|
||||
|
||||
async function harness(configOverrides: Partial<FreewayConfig> = {}, env: Record<string, string> = {}): Promise<Harness> {
|
||||
const alpha = (await createMockUpstream({ name: 'alpha' })) as Mock;
|
||||
const beta = (await createMockUpstream({ name: 'beta' })) as Mock;
|
||||
cleanups.push(() => alpha.close(), () => beta.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-gw-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
writeFileSync(
|
||||
join(dir, 'alpha.json'),
|
||||
JSON.stringify({
|
||||
id: 'alpha',
|
||||
priority: 10,
|
||||
baseUrl: alpha.url,
|
||||
auth: { type: 'bearer', envKeys: ['ALPHA_KEY'] },
|
||||
limits: { rpm: 100 },
|
||||
models: [
|
||||
{ id: 'alpha-chat', alias: ['fast'], context: 128000, caps: ['chat', 'tools', 'json'], priority: 10 },
|
||||
{ id: 'alpha-embed', alias: ['embed'], context: 8000, caps: ['embed'] },
|
||||
],
|
||||
}),
|
||||
);
|
||||
writeFileSync(
|
||||
join(dir, 'beta.json'),
|
||||
JSON.stringify({
|
||||
id: 'beta',
|
||||
priority: 20,
|
||||
baseUrl: beta.url,
|
||||
auth: { type: 'bearer', envKeys: ['BETA_KEY'] },
|
||||
limits: { rpm: 100 },
|
||||
models: [{ id: '@cf/meta/llama-3.1-8b-instruct', alias: ['fast'], context: 8192, caps: ['chat'], priority: 10 }],
|
||||
}),
|
||||
);
|
||||
|
||||
const registry = new Registry({
|
||||
dir,
|
||||
env: { ALPHA_KEY: 'ak-000000000001,ak-000000000002', BETA_KEY: 'bk-000000000001', ...env },
|
||||
logger: silentLogger,
|
||||
});
|
||||
const store = new Store({ file: null, cooldownMs: 60_000, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
Object.assign(config, configOverrides);
|
||||
const router = new Router(registry, store, config);
|
||||
|
||||
const server: Server = createGateway({ registry, store, router, config, logger: silentLogger });
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
return { base: `http://127.0.0.1:${port}`, alpha, beta, store, registry, config };
|
||||
}
|
||||
|
||||
async function post(base: string, path: string, body: unknown, headers: Record<string, string> = {}): Promise<Response> {
|
||||
return fetch(`${base}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', ...headers },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
const CHAT = { model: 'auto', messages: [{ role: 'user', content: 'hello there' }] };
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('gateway — a normal completion', () => {
|
||||
it('proxies the request and reports who served it', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', CHAT);
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha', 'priority order picked alpha');
|
||||
assert.equal(res.headers.get('x-freeway-model'), 'alpha-chat');
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '1');
|
||||
assert.equal(res.headers.get('x-freeway-route'), 'auto');
|
||||
assert.ok(Number(res.headers.get('x-freeway-ms')) >= 0);
|
||||
|
||||
const body = (await res.json()) as { choices: { message: { content: string } }[] };
|
||||
assert.match(body.choices[0]?.message.content ?? '', /\[alpha\]/);
|
||||
});
|
||||
|
||||
it('never forwards the virtual key upstream, only the provider key', async () => {
|
||||
const h = await harness({ keys: ['fw-secret-key'] });
|
||||
await post(h.base, '/v1/chat/completions', CHAT, { authorization: 'Bearer fw-secret-key' });
|
||||
|
||||
const call = h.alpha.callsTo('/chat/completions')[0];
|
||||
assert.ok(call);
|
||||
// Which key of the pool served it depends on cursor rotation; what matters
|
||||
// is that it is a provider key and never the caller's virtual one.
|
||||
assert.ok(['ak-000000000001', 'ak-000000000002'].includes(call.key ?? ''), `unexpected upstream key ${call.key}`);
|
||||
assert.notEqual(call.key, 'fw-secret-key');
|
||||
});
|
||||
|
||||
it('rewrites the model to the upstream id', async () => {
|
||||
const h = await harness();
|
||||
await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
const call = h.alpha.callsTo('/chat/completions')[0];
|
||||
assert.equal((call?.body as { model: string }).model, 'alpha-chat');
|
||||
});
|
||||
|
||||
it('records tokens from the upstream usage block', async () => {
|
||||
const h = await harness();
|
||||
await post(h.base, '/v1/chat/completions', CHAT);
|
||||
assert.ok(h.store.provider('alpha').meter.used('tpm') > 0, 'usage was metered');
|
||||
});
|
||||
|
||||
it('falls back to an estimate when a provider reports no usage', async () => {
|
||||
// A provider reporting nothing must not leave the quota counter reading zero.
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 200, dropUsage: true, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', CHAT);
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-usage'), 'estimated');
|
||||
assert.ok(h.store.provider('alpha').meter.used('tpm') > 0, 'estimated tokens still count');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — failover', () => {
|
||||
it('a 429 on provider A fails over to provider B', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'beta');
|
||||
// 3, not 2: a 429 is a key fault, so alpha's *other* pool key is tried
|
||||
// before the provider is abandoned. That is the point of a key pool.
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '3');
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, 2, 'both alpha keys were tried');
|
||||
|
||||
const body = (await res.json()) as { choices: { message: { content: string } }[] };
|
||||
assert.match(body.choices[0]?.message.content ?? '', /\[beta\]/);
|
||||
});
|
||||
|
||||
it('a 429 on one key retries the other key of the same provider', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.setKeys({ 'ak-000000000001': { status: 429 } });
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'alpha/alpha-chat' });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha', 'stayed on the same provider');
|
||||
|
||||
const keysUsed = h.alpha.callsTo('/chat/completions').map((c) => c.key);
|
||||
assert.ok(keysUsed.includes('ak-000000000002'), 'the pool’s other key served it');
|
||||
});
|
||||
|
||||
it('a 5xx benches the provider so the next request skips it entirely', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 503, repeat: Infinity }]);
|
||||
|
||||
await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
const before = h.alpha.callsTo('/chat/completions').length;
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'beta');
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '1', 'the benched provider is not even tried');
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, before, 'no second call to the dead provider');
|
||||
});
|
||||
|
||||
it('honours an upstream Retry-After over the configured cooldown', async () => {
|
||||
const h = await harness({ cooldownSeconds: 600 });
|
||||
h.alpha.script([{ status: 429, headers: { 'retry-after': '2' }, repeat: Infinity }]);
|
||||
|
||||
await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'alpha/alpha-chat' });
|
||||
const cooldown = h.store.key('alpha', 'alpha#0').cooldownUntil - Date.now();
|
||||
assert.ok(cooldown <= 2500 && cooldown > 0, `expected ~2s from Retry-After, got ${cooldown}ms`);
|
||||
});
|
||||
|
||||
it('stops after maxAttempts rather than walking every candidate', async () => {
|
||||
const h = await harness({ maxAttempts: 1 });
|
||||
h.alpha.script([{ status: 500, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.status, 502);
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '1');
|
||||
assert.equal(h.beta.callsTo('/chat/completions').length, 0, 'never got to beta');
|
||||
});
|
||||
|
||||
it('returns a 400 immediately without retrying anywhere', async () => {
|
||||
// A malformed body fails identically everywhere; retrying burns quota for nothing.
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 400, body: { error: { message: 'bad parameter foo', type: 'invalid_request_error' } }, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
|
||||
assert.equal(res.status, 400);
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '1');
|
||||
assert.equal(h.beta.callsTo('/chat/completions').length, 0, 'beta was never tried');
|
||||
|
||||
const body = (await res.json()) as { error: { message: string } };
|
||||
assert.equal(body.error.message, 'bad parameter foo', 'the upstream error is passed through verbatim');
|
||||
assert.equal(h.store.isProviderCooling('alpha'), false, 'a 400 benches nothing');
|
||||
});
|
||||
|
||||
it('reports every attempt when they all fail', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 500, repeat: Infinity }]);
|
||||
h.beta.script([{ status: 502, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.status, 502);
|
||||
|
||||
const body = (await res.json()) as { freeway: { attempts: { provider: string; status: number }[] } };
|
||||
// A 500 is a provider fault, so alpha's second key is skipped — every key
|
||||
// would fail identically and the attempt budget is better spent on beta.
|
||||
assert.equal(body.freeway.attempts.length, 2);
|
||||
assert.deepEqual(body.freeway.attempts.map((a) => a.provider), ['alpha', 'beta']);
|
||||
assert.deepEqual(body.freeway.attempts.map((a) => a.status), [500, 502]);
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, 1, 'the benched provider was not retried on another key');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — streaming', () => {
|
||||
it('proxies SSE through intact', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, stream: true });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.match(res.headers.get('content-type') ?? '', /text\/event-stream/);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha');
|
||||
|
||||
const text = await res.text();
|
||||
assert.match(text, /^data: /m);
|
||||
assert.match(text, /data: \[DONE\]/);
|
||||
|
||||
// Reassemble the deltas — the content must survive the transform untouched.
|
||||
const content = [...text.matchAll(/^data: (.+)$/gm)]
|
||||
.map((m) => m[1])
|
||||
.filter((p): p is string => p !== undefined && p !== '[DONE]')
|
||||
.map((p) => JSON.parse(p) as { choices?: { delta?: { content?: string } }[] })
|
||||
.flatMap((o) => o.choices ?? [])
|
||||
.map((c) => c.delta?.content ?? '')
|
||||
.join('');
|
||||
assert.match(content, /\[alpha\]/);
|
||||
});
|
||||
|
||||
it('accounts for tokens from the final usage chunk of a stream', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, stream: true });
|
||||
await res.text();
|
||||
|
||||
// Usage arrives only in the last chunk, long after the response committed.
|
||||
assert.ok(h.store.provider('alpha').meter.used('tpm') > 0, 'late usage was still metered');
|
||||
const log = h.store.logs(1)[0];
|
||||
assert.ok(log);
|
||||
assert.equal(log.stream, true);
|
||||
assert.ok(log.tokensOut > 0);
|
||||
});
|
||||
|
||||
it('still fails over when the first provider 429s a stream request', async () => {
|
||||
// The commit point is after 200 headers, so a pre-body failure is recoverable.
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast', stream: true });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'beta');
|
||||
assert.match(await res.text(), /\[beta\]/);
|
||||
});
|
||||
|
||||
it('estimates stream tokens when the provider omits usage', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 200, dropUsage: true, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, stream: true });
|
||||
await res.text();
|
||||
assert.ok(h.store.provider('alpha').meter.used('tpm') > 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — quota exhaustion', () => {
|
||||
it('returns 429 with a diagnostic reason per provider', async () => {
|
||||
const h = await harness();
|
||||
for (let i = 0; i < 100; i++) {
|
||||
h.store.recordSuccess('alpha', 'alpha#0', 5);
|
||||
h.store.recordSuccess('beta', 'beta#0', 5);
|
||||
}
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.status, 429);
|
||||
assert.ok(Number(res.headers.get('retry-after')) > 0, 'tells the client when to come back');
|
||||
|
||||
const body = (await res.json()) as {
|
||||
error: { code: string };
|
||||
freeway: { blocked: { provider: string; reason: string }[]; retryAfterMs: number };
|
||||
};
|
||||
assert.equal(body.error.code, 'all_providers_blocked');
|
||||
assert.equal(body.freeway.blocked.length, 2);
|
||||
for (const b of body.freeway.blocked) {
|
||||
assert.match(b.reason, /rpm limit reached \(100\/100 last 60s\)/, `reason must be specific, got "${b.reason}"`);
|
||||
}
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, 0, 'no upstream call was wasted');
|
||||
});
|
||||
|
||||
it('tells you which env var to set when nothing is configured', async () => {
|
||||
const h = await harness({}, { ALPHA_KEY: '', BETA_KEY: '' });
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
|
||||
assert.equal(res.status, 429);
|
||||
const body = (await res.json()) as { freeway: { blocked: { reason: string }[] } };
|
||||
const reasons = body.freeway.blocked.map((b) => b.reason).join(' ');
|
||||
assert.match(reasons, /ALPHA_KEY/);
|
||||
assert.match(reasons, /BETA_KEY/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — unknown models', () => {
|
||||
it('returns 404 with the closest available ids', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'gpt-4o-turbo-max' });
|
||||
|
||||
assert.equal(res.status, 404);
|
||||
const body = (await res.json()) as { error: { code: string }; freeway: { suggestions: string[] } };
|
||||
assert.equal(body.error.code, 'model_not_found');
|
||||
assert.ok(Array.isArray(body.freeway.suggestions));
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — virtual keys', () => {
|
||||
it('rejects a request with no key when keys are configured', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
const res = await post(h.base, '/v1/chat/completions', CHAT);
|
||||
|
||||
assert.equal(res.status, 401);
|
||||
const body = (await res.json()) as { error: { code: string; message: string } };
|
||||
assert.equal(body.error.code, 'invalid_api_key');
|
||||
assert.match(body.error.message, /missing API key/);
|
||||
});
|
||||
|
||||
it('rejects a wrong key', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
const res = await post(h.base, '/v1/chat/completions', CHAT, { authorization: 'Bearer fw-wrong' });
|
||||
assert.equal(res.status, 401);
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, 0, 'never reached an upstream');
|
||||
});
|
||||
|
||||
it('accepts a correct key, via Authorization or x-api-key', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
assert.equal((await post(h.base, '/v1/chat/completions', CHAT, { authorization: 'Bearer fw-good' })).status, 200);
|
||||
assert.equal((await post(h.base, '/v1/chat/completions', CHAT, { 'x-api-key': 'fw-good' })).status, 200);
|
||||
});
|
||||
|
||||
it('is open when no keys are configured', async () => {
|
||||
const h = await harness();
|
||||
assert.equal((await post(h.base, '/v1/chat/completions', CHAT)).status, 200);
|
||||
});
|
||||
|
||||
it('leaves /healthz reachable without a key', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
const res = await fetch(`${h.base}/healthz`);
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(((await res.json()) as { ok: boolean }).ok, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — embeddings', () => {
|
||||
it('routes to an embedding model, never a chat one', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/embeddings', { model: 'auto', input: ['alpha', 'beta'] });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-model'), 'alpha-embed');
|
||||
|
||||
const body = (await res.json()) as { data: { embedding: number[] }[] };
|
||||
assert.equal(body.data.length, 2);
|
||||
assert.ok(Array.isArray(body.data[0]?.embedding));
|
||||
});
|
||||
|
||||
it('rejects an embeddings call with no input', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/embeddings', { model: 'auto' });
|
||||
assert.equal(res.status, 400);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — capability routing', () => {
|
||||
it('sends a tools request only to a provider that supports tools', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', {
|
||||
...CHAT,
|
||||
model: 'fast',
|
||||
tools: [{ type: 'function', function: { name: 'get_weather', parameters: {} } }],
|
||||
});
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha', 'beta lacks the tools capability');
|
||||
assert.equal(h.beta.callsTo('/chat/completions').length, 0);
|
||||
});
|
||||
|
||||
it('routes a json_object request to a json-capable model', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast', response_format: { type: 'json_object' } });
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — dashboard API', () => {
|
||||
let h: Harness;
|
||||
beforeEach(async () => {
|
||||
h = await harness();
|
||||
});
|
||||
|
||||
it('GET /v1/models returns provider/model ids plus virtual aliases', async () => {
|
||||
const res = await fetch(`${h.base}/v1/models`);
|
||||
assert.equal(res.status, 200);
|
||||
|
||||
const body = (await res.json()) as { object: string; data: { id: string; owned_by: string }[] };
|
||||
assert.equal(body.object, 'list');
|
||||
const ids = body.data.map((d) => d.id);
|
||||
assert.ok(ids.includes('alpha/alpha-chat'));
|
||||
assert.ok(ids.includes('beta/@cf/meta/llama-3.1-8b-instruct'), 'slashy ids survive the catalog');
|
||||
assert.ok(ids.includes('fast'), 'aliases appear as virtual models');
|
||||
assert.ok(ids.includes('auto'));
|
||||
});
|
||||
|
||||
it('GET /api/providers exposes state with masked keys only', async () => {
|
||||
const res = await fetch(`${h.base}/api/providers`);
|
||||
const body = (await res.json()) as { providers: { id: string; keys: { masked: string }[]; bars: unknown[] }[] };
|
||||
|
||||
const alpha = body.providers.find((p) => p.id === 'alpha');
|
||||
assert.ok(alpha);
|
||||
assert.equal(alpha.keys.length, 2);
|
||||
assert.equal(alpha.keys[0]?.masked, 'ak-0…0001');
|
||||
|
||||
const raw = JSON.stringify(body);
|
||||
assert.ok(!raw.includes('ak-000000000001'), 'a raw provider key must never reach the dashboard');
|
||||
assert.ok(!raw.includes('bk-000000000001'));
|
||||
});
|
||||
|
||||
it('GET /api/logs shows the served request', async () => {
|
||||
await post(h.base, '/v1/chat/completions', CHAT);
|
||||
const res = await fetch(`${h.base}/api/logs`);
|
||||
const body = (await res.json()) as { logs: { providerId: string; status: number; resolution: string }[] };
|
||||
|
||||
assert.equal(body.logs.length, 1);
|
||||
assert.equal(body.logs[0]?.providerId, 'alpha');
|
||||
assert.equal(body.logs[0]?.status, 200);
|
||||
assert.equal(body.logs[0]?.resolution, 'auto');
|
||||
});
|
||||
|
||||
it('POST /api/providers/:id/enabled takes a provider out of rotation', async () => {
|
||||
const toggle = await post(h.base, '/api/providers/alpha/enabled', { enabled: false });
|
||||
assert.equal(toggle.status, 200);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'beta');
|
||||
});
|
||||
|
||||
it('POST /api/reload re-reads providers without a restart', async () => {
|
||||
const res = await post(h.base, '/api/reload', {});
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(((await res.json()) as { ok: boolean }).ok, true);
|
||||
});
|
||||
|
||||
it('GET /api/stats summarises the fleet', async () => {
|
||||
const res = await fetch(`${h.base}/api/stats`);
|
||||
const body = (await res.json()) as { providers: { total: number; usable: number }; strategy: string };
|
||||
assert.equal(body.providers.total, 2);
|
||||
assert.equal(body.providers.usable, 2);
|
||||
assert.equal(body.strategy, 'priority');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — live event stream', () => {
|
||||
/** Read SSE frames off a fetch body until `want` events arrive or time runs out. */
|
||||
async function collect(res: Response, want: number, timeoutMs = 4000): Promise<{ event: string; data: Record<string, unknown> }[]> {
|
||||
const out: { event: string; data: Record<string, unknown> }[] = [];
|
||||
const reader = res.body!.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let event = 'message';
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
|
||||
while (out.length < want && Date.now() < deadline) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
let nl: number;
|
||||
while ((nl = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, nl).trim();
|
||||
buffer = buffer.slice(nl + 1);
|
||||
if (line.startsWith('event:')) event = line.slice(6).trim();
|
||||
else if (line.startsWith('data:')) {
|
||||
out.push({ event, data: JSON.parse(line.slice(5).trim()) as Record<string, unknown> });
|
||||
event = 'message';
|
||||
}
|
||||
}
|
||||
}
|
||||
await reader.cancel().catch(() => {});
|
||||
return out;
|
||||
}
|
||||
|
||||
it('pushes a log entry the moment a request completes', async () => {
|
||||
const h = await harness();
|
||||
const res = await fetch(`${h.base}/api/events`, { headers: { accept: 'text/event-stream' } });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.match(res.headers.get('content-type') ?? '', /text\/event-stream/);
|
||||
|
||||
const events = collect(res, 2);
|
||||
await post(h.base, '/v1/chat/completions', CHAT);
|
||||
const got = await events;
|
||||
|
||||
assert.equal(got[0]?.event, 'hello');
|
||||
const log = got.find((e) => e.event === 'log');
|
||||
assert.ok(log, 'the completed request arrived as a live event');
|
||||
assert.equal(log.data['providerId'], 'alpha');
|
||||
assert.equal(log.data['status'], 200);
|
||||
assert.equal(log.data['resolution'], 'auto');
|
||||
});
|
||||
|
||||
it('pushes failures too, with their attempt trail', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 429, repeat: Infinity }]);
|
||||
const res = await fetch(`${h.base}/api/events`, { headers: { accept: 'text/event-stream' } });
|
||||
|
||||
const events = collect(res, 2);
|
||||
await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
const log = (await events).find((e) => e.event === 'log');
|
||||
|
||||
assert.ok(log);
|
||||
assert.equal(log.data['providerId'], 'beta');
|
||||
assert.ok(Array.isArray(log.data['attempts']) && (log.data['attempts'] as unknown[]).length > 0);
|
||||
});
|
||||
|
||||
it('unsubscribes when the client goes away, so listeners cannot leak', async () => {
|
||||
const h = await harness();
|
||||
assert.equal(h.store.listenerCount(), 0);
|
||||
|
||||
const controller = new AbortController();
|
||||
const res = await fetch(`${h.base}/api/events`, { signal: controller.signal });
|
||||
await collect(res, 1); // wait for `hello` so the subscription is registered
|
||||
assert.equal(h.store.listenerCount(), 1);
|
||||
|
||||
controller.abort();
|
||||
// A long-lived dashboard reconnecting all day must not accumulate listeners.
|
||||
for (let i = 0; i < 40 && h.store.listenerCount() > 0; i++) await new Promise((r) => setTimeout(r, 25));
|
||||
assert.equal(h.store.listenerCount(), 0);
|
||||
});
|
||||
|
||||
it('requires a virtual key like the rest of /api', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
assert.equal((await fetch(`${h.base}/api/events`)).status, 401);
|
||||
const ok = await fetch(`${h.base}/api/events`, { headers: { authorization: 'Bearer fw-good' } });
|
||||
assert.equal(ok.status, 200);
|
||||
await ok.body?.cancel();
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — dashboard', () => {
|
||||
it('serves the dashboard at / without requiring a key', async () => {
|
||||
// The dashboard has to load before it can ask for a key, so `/` itself is
|
||||
// never gated; its `/api/*` fetches are.
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
const res = await fetch(`${h.base}/`);
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.match(res.headers.get('content-type') ?? '', /text\/html/);
|
||||
const html = await res.text();
|
||||
assert.match(html, /<title>Freeway<\/title>/);
|
||||
});
|
||||
|
||||
it('is entirely self-contained — no CDN, no external anything', async () => {
|
||||
// A dashboard that phones out is a dashboard that breaks on an air-gapped
|
||||
// box and leaks which providers you run to whoever hosts the asset.
|
||||
const h = await harness();
|
||||
const html = await (await fetch(`${h.base}/`)).text();
|
||||
|
||||
const external = [...html.matchAll(/\b(?:src|href)\s*=\s*["']([^"']+)["']/gi)]
|
||||
.map((m) => m[1] ?? '')
|
||||
.filter((url) => /^(https?:)?\/\//i.test(url));
|
||||
|
||||
assert.deepEqual(external, [], `dashboard must not reference external assets, found: ${external.join(', ')}`);
|
||||
assert.ok(!/@import/i.test(html), 'no CSS @import either');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — malformed input', () => {
|
||||
it('rejects a body that is not JSON', async () => {
|
||||
const h = await harness();
|
||||
const res = await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: '{ not json',
|
||||
});
|
||||
assert.equal(res.status, 400);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'invalid_json');
|
||||
});
|
||||
|
||||
it('rejects a chat request with no messages', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { model: 'auto' });
|
||||
assert.equal(res.status, 400);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'missing_messages');
|
||||
});
|
||||
|
||||
it('rejects a body over the size cap with 413, not a dropped connection', async () => {
|
||||
const { config: defaults } = parseConfig({}, {});
|
||||
const h = await harness({ guard: { ...defaults.guard, maxBodyBytes: 512 } });
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', {
|
||||
model: 'auto',
|
||||
messages: [{ role: 'user', content: 'x'.repeat(5000) }],
|
||||
});
|
||||
assert.equal(res.status, 413);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'payload_too_large');
|
||||
});
|
||||
|
||||
it('404s an unknown route', async () => {
|
||||
const h = await harness();
|
||||
const res = await fetch(`${h.base}/v1/nonsense`);
|
||||
assert.equal(res.status, 404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,263 @@
|
||||
/**
|
||||
* Regression tests for the security sweep.
|
||||
*
|
||||
* Each of these encodes a vulnerability that was real, so a future refactor
|
||||
* cannot quietly reintroduce it.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { ContextStore, ExactCache, Registry, Router, Store, Tokenizer, parseConfig, silentLogger, type FreewayConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from '../src/server.ts';
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock { url: string; close(): Promise<unknown> }
|
||||
|
||||
interface Harness {
|
||||
base: string;
|
||||
context: ContextStore;
|
||||
dataDir: string;
|
||||
}
|
||||
|
||||
async function harness(over: Partial<FreewayConfig> = {}): Promise<Harness> {
|
||||
const upstream = (await createMockUpstream({ name: 'up' })) as Mock;
|
||||
cleanups.push(() => upstream.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-sec-'));
|
||||
const dataDir = mkdtempSync(join(tmpdir(), 'freeway-sec-data-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }), () => rmSync(dataDir, { recursive: true, force: true }));
|
||||
|
||||
writeFileSync(join(dir, 'up.json'), JSON.stringify({
|
||||
id: 'up', priority: 10, baseUrl: upstream.url,
|
||||
auth: { type: 'bearer', envKeys: ['UP_KEY'] }, limits: { rpm: 1000 },
|
||||
models: [{ id: 'up-chat', alias: ['fast'], context: 128000, caps: ['chat'] }],
|
||||
}));
|
||||
|
||||
const registry = new Registry({ dir, env: { UP_KEY: 'uk-000000000001' }, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
Object.assign(config, over);
|
||||
const router = new Router(registry, store, config);
|
||||
|
||||
const context = new ContextStore(join(dataDir, 'freeway.db'));
|
||||
const cache = new ExactCache({ file: join(dataDir, 'cache.db'), mode: 'safe', ttlMs: 3_600_000, maxEntries: 50 });
|
||||
cleanups.push(() => context.close(), () => cache.close());
|
||||
|
||||
const server: Server = createGateway({
|
||||
registry, store, router, config, logger: silentLogger,
|
||||
contextStore: context, tokenizer: new Tokenizer(), cache,
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
return { base: `http://127.0.0.1:${port}`, context, dataDir };
|
||||
}
|
||||
|
||||
const APP = 'fw-app-key';
|
||||
const ADMIN = 'fw-admin-key';
|
||||
const authed = (key: string) => ({ 'content-type': 'application/json', authorization: `Bearer ${key}` });
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('C1 — an inference key is not an admin key', () => {
|
||||
const cfg = { keys: [APP], adminKey: ADMIN, host: '0.0.0.0' };
|
||||
|
||||
it('refuses an app key on every /api route', async () => {
|
||||
const h = await harness(cfg);
|
||||
const routes: [string, string][] = [
|
||||
['GET', '/api/providers'], ['GET', '/api/logs'], ['GET', '/api/sessions'],
|
||||
['GET', '/api/alerts'], ['GET', '/api/cache'], ['GET', '/api/stats'], ['GET', '/api/events'],
|
||||
['POST', '/api/reload'], ['POST', '/api/providers/up/enabled'],
|
||||
['POST', '/api/providers/up/key'], ['POST', '/api/providers/up/probe'],
|
||||
['DELETE', '/api/cache'],
|
||||
];
|
||||
|
||||
for (const [method, path] of routes) {
|
||||
const res = await fetch(`${h.base}${path}`, { method, headers: authed(APP), ...(method === 'GET' ? {} : { body: '{}' }) });
|
||||
assert.equal(res.status, 403, `${method} ${path} should be forbidden to an app key`);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'admin_required');
|
||||
}
|
||||
});
|
||||
|
||||
it('still lets the app key do inference', async () => {
|
||||
const h = await harness(cfg);
|
||||
const res = await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: authed(APP),
|
||||
body: JSON.stringify({ model: 'fast', messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
assert.equal(res.status, 200);
|
||||
});
|
||||
|
||||
it('lets the admin key through', async () => {
|
||||
const h = await harness(cfg);
|
||||
assert.equal((await fetch(`${h.base}/api/stats`, { headers: authed(ADMIN) })).status, 200);
|
||||
});
|
||||
|
||||
it('refuses /api entirely on a public bind with no admin key', async () => {
|
||||
// Otherwise an exposed gateway with only inference keys hands them the box.
|
||||
const h = await harness({ keys: [APP], adminKey: null, host: '0.0.0.0' });
|
||||
const res = await fetch(`${h.base}/api/logs`, { headers: authed(APP) });
|
||||
assert.equal(res.status, 403);
|
||||
assert.match(((await res.json()) as { error: { message: string } }).error.message, /FREEWAY_ADMIN_KEY/);
|
||||
});
|
||||
|
||||
it('stays convenient on loopback with no admin key', async () => {
|
||||
const h = await harness({ keys: [APP], adminKey: null, host: '127.0.0.1' });
|
||||
assert.equal((await fetch(`${h.base}/api/stats`, { headers: authed(APP) })).status, 200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('C2 — CORS is an allowlist, never a wildcard', () => {
|
||||
it('sends no CORS headers by default', async () => {
|
||||
const h = await harness({ keys: [APP] });
|
||||
const res = await fetch(`${h.base}/v1/models`, { headers: { ...authed(APP), origin: 'https://evil.example' } });
|
||||
assert.equal(res.headers.get('access-control-allow-origin'), null);
|
||||
});
|
||||
|
||||
it('refuses an origin that is not on the list', async () => {
|
||||
const h = await harness({ keys: [APP], corsOrigins: ['https://app.example'] });
|
||||
const res = await fetch(`${h.base}/v1/models`, { headers: { ...authed(APP), origin: 'https://evil.example' } });
|
||||
assert.equal(res.headers.get('access-control-allow-origin'), null);
|
||||
});
|
||||
|
||||
it('echoes only a listed origin, and varies on it', async () => {
|
||||
const h = await harness({ keys: [APP], corsOrigins: ['https://app.example'] });
|
||||
const res = await fetch(`${h.base}/v1/models`, { headers: { ...authed(APP), origin: 'https://app.example' } });
|
||||
assert.equal(res.headers.get('access-control-allow-origin'), 'https://app.example');
|
||||
assert.match(res.headers.get('vary') ?? '', /Origin/);
|
||||
});
|
||||
|
||||
it('rejects "*" in configuration rather than honouring it', () => {
|
||||
const { config, issues } = parseConfig({ corsOrigins: ['*', 'https://ok.example'] }, {});
|
||||
assert.deepEqual(config.corsOrigins, ['https://ok.example']);
|
||||
assert.ok(issues.some((i) => i.path === 'corsOrigins'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('C3 — sessions belong to the key that made them', () => {
|
||||
it('refuses to read another key’s session', async () => {
|
||||
const h = await harness({ keys: [APP, 'fw-other-key'], adminKey: ADMIN });
|
||||
|
||||
await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: authed(APP),
|
||||
body: JSON.stringify({ model: 'fast', session: 'private', messages: [{ role: 'user', content: 'my bank pin is 1234' }] }),
|
||||
});
|
||||
|
||||
const res = await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: authed('fw-other-key'),
|
||||
body: JSON.stringify({ model: 'fast', session: 'private', messages: [{ role: 'user', content: 'what did I say?' }] }),
|
||||
});
|
||||
|
||||
assert.equal(res.status, 403);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'session_forbidden');
|
||||
});
|
||||
|
||||
it('lets the owner continue its own session', async () => {
|
||||
const h = await harness({ keys: [APP], adminKey: ADMIN });
|
||||
const body = (content: string) => JSON.stringify({ model: 'fast', session: 'mine', messages: [{ role: 'user', content }] });
|
||||
|
||||
assert.equal((await fetch(`${h.base}/v1/chat/completions`, { method: 'POST', headers: authed(APP), body: body('one') })).status, 200);
|
||||
assert.equal((await fetch(`${h.base}/v1/chat/completions`, { method: 'POST', headers: authed(APP), body: body('two') })).status, 200);
|
||||
assert.equal(h.context.messages('mine').length, 4);
|
||||
});
|
||||
|
||||
it('records the owner, masked', async () => {
|
||||
const h = await harness({ keys: [APP], adminKey: ADMIN });
|
||||
await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: authed(APP),
|
||||
body: JSON.stringify({ model: 'fast', session: 'owned', messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
const owner = h.context.info('owned')?.owner;
|
||||
assert.ok(owner, 'session should be claimed');
|
||||
assert.ok(!owner.includes('app-key'), 'the owner must be a mask, not the key');
|
||||
});
|
||||
});
|
||||
|
||||
describe('H2 — live log subscriptions are capped', () => {
|
||||
it('refuses beyond the cap instead of accumulating listeners', async () => {
|
||||
const h = await harness();
|
||||
const controllers: AbortController[] = [];
|
||||
let refused = 0;
|
||||
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const controller = new AbortController();
|
||||
controllers.push(controller);
|
||||
const res = await fetch(`${h.base}/api/events`, { signal: controller.signal });
|
||||
if (res.status === 503) {
|
||||
refused += 1;
|
||||
break;
|
||||
}
|
||||
// Start reading so the subscription actually registers server-side.
|
||||
void res.body?.getReader().read();
|
||||
}
|
||||
for (const c of controllers) c.abort();
|
||||
assert.ok(refused > 0, 'an unbounded number of subscribers was accepted');
|
||||
});
|
||||
});
|
||||
|
||||
describe('H3 — response headers cannot be poisoned by input', () => {
|
||||
it('survives a session id full of illegal header bytes', async () => {
|
||||
// These used to reach writeHead raw and throw, turning a good response
|
||||
// into a remote 500.
|
||||
const h = await harness();
|
||||
for (const id of ['sess\r\nX-Injected: yes', 'sess→arrow', 'sess null']) {
|
||||
const res = await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model: 'fast', session: id, messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
assert.equal(res.status, 200, `session id ${JSON.stringify(id)} should not break the response`);
|
||||
assert.equal(res.headers.get('x-injected'), null, 'no header was injected');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('M1 — data at rest is not world-readable', () => {
|
||||
it('creates the sqlite files 0600', async () => {
|
||||
const h = await harness();
|
||||
await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model: 'fast', session: 's', temperature: 0, messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
|
||||
for (const file of ['freeway.db', 'cache.db']) {
|
||||
const mode = statSync(join(h.dataDir, file)).mode & 0o777;
|
||||
assert.equal(mode, 0o600, `${file} is 0${mode.toString(8)}, expected 0600`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('M2 — key comparison leaks neither content nor length', () => {
|
||||
it('rejects keys of every length identically', async () => {
|
||||
const h = await harness({ keys: ['fw-the-real-key-abcdefghijk'] });
|
||||
for (const wrong of ['x', 'fw-', 'fw-the-real-key-abcdefghij', 'fw-the-real-key-abcdefghijkX', 'x'.repeat(500)]) {
|
||||
const res = await fetch(`${h.base}/v1/models`, { headers: { authorization: `Bearer ${wrong}` } });
|
||||
assert.equal(res.status, 401, `"${wrong.slice(0, 12)}…" should be rejected`);
|
||||
}
|
||||
assert.equal((await fetch(`${h.base}/v1/models`, { headers: authed('fw-the-real-key-abcdefghijk') })).status, 200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('still no way to reach a raw provider key', () => {
|
||||
it('keeps provider keys out of every admin response', async () => {
|
||||
const h = await harness({ keys: [APP], adminKey: ADMIN });
|
||||
for (const path of ['/api/providers', '/api/stats', '/api/logs', '/api/alerts']) {
|
||||
const text = await (await fetch(`${h.base}${path}`, { headers: authed(ADMIN) })).text();
|
||||
assert.ok(!text.includes('uk-000000000001'), `${path} leaked a provider key`);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* End-to-end sessions: the client sends one message, the gateway rebuilds the
|
||||
* thread, and the thread survives a failover to a much smaller model.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { ContextStore, Registry, Router, Store, Tokenizer, parseConfig, silentLogger, type FreewayConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from '../src/server.ts';
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock {
|
||||
url: string;
|
||||
script(steps: unknown[]): void;
|
||||
callsTo(path: string): { body: { messages?: { role: string; content: string }[]; model?: string } }[];
|
||||
close(): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
base: string;
|
||||
big: Mock;
|
||||
small: Mock;
|
||||
context: ContextStore;
|
||||
store: Store;
|
||||
}
|
||||
|
||||
/** `big` has a 128k window, `small` has 8k — the shape of the real scenario. */
|
||||
async function harness(over: Partial<FreewayConfig> = {}): Promise<Harness> {
|
||||
const big = (await createMockUpstream({ name: 'big' })) as Mock;
|
||||
const small = (await createMockUpstream({ name: 'small' })) as Mock;
|
||||
cleanups.push(() => big.close(), () => small.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-sess-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
writeFileSync(join(dir, 'big.json'), JSON.stringify({
|
||||
id: 'big', priority: 10, baseUrl: big.url,
|
||||
auth: { type: 'bearer', envKeys: ['BIG_KEY'] }, limits: { rpm: 100 },
|
||||
models: [{ id: 'big-chat', alias: ['fast'], context: 128000, caps: ['chat'] }],
|
||||
}));
|
||||
writeFileSync(join(dir, 'small.json'), JSON.stringify({
|
||||
id: 'small', priority: 20, baseUrl: small.url,
|
||||
auth: { type: 'bearer', envKeys: ['SMALL_KEY'] }, limits: { rpm: 100 },
|
||||
models: [{ id: 'small-chat', alias: ['fast'], context: 8192, caps: ['chat'] }],
|
||||
}));
|
||||
|
||||
const registry = new Registry({ dir, env: { BIG_KEY: 'bk-000000000001', SMALL_KEY: 'sk-000000000001' }, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
Object.assign(config, over);
|
||||
const router = new Router(registry, store, config);
|
||||
const context = new ContextStore(':memory:');
|
||||
cleanups.push(() => context.close());
|
||||
|
||||
const server: Server = createGateway({
|
||||
registry, store, router, config, logger: silentLogger,
|
||||
contextStore: context, tokenizer: new Tokenizer(),
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
return { base: `http://127.0.0.1:${port}`, big, small, context, store };
|
||||
}
|
||||
|
||||
async function chat(base: string, body: unknown): Promise<Response> {
|
||||
return fetch(`${base}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
const say = (text: string) => ({ role: 'user', content: text });
|
||||
|
||||
describe('sessions — server-side conversation', () => {
|
||||
it('rebuilds the thread from just the new message', async () => {
|
||||
const h = await harness();
|
||||
|
||||
await chat(h.base, { model: 'fast', session: 'thread-1', messages: [say('my name is Ada')] });
|
||||
const res = await chat(h.base, { model: 'fast', session: 'thread-1', messages: [say('what is my name?')] });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-session'), 'thread-1');
|
||||
|
||||
// The upstream saw the whole conversation, not just the last line.
|
||||
const calls = h.big.callsTo('/chat/completions');
|
||||
const sent = calls[calls.length - 1]?.body.messages ?? [];
|
||||
assert.ok(sent.length >= 3, `expected rehydrated history, got ${sent.length} messages`);
|
||||
assert.ok(sent.some((m) => m.content.includes('my name is Ada')), 'the earlier turn was replayed');
|
||||
});
|
||||
|
||||
it('strips `session` before forwarding — no provider ever sees it', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, { model: 'fast', session: 'thread-2', messages: [say('hi')] });
|
||||
|
||||
const body = h.big.callsTo('/chat/completions')[0]?.body as Record<string, unknown>;
|
||||
assert.equal(body['session'], undefined);
|
||||
});
|
||||
|
||||
it('persists the assistant reply so the next turn sees it', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, { model: 'fast', session: 'thread-3', messages: [say('hello')] });
|
||||
|
||||
const stored = h.context.messages('thread-3');
|
||||
assert.deepEqual(stored.map((m) => m.role), ['user', 'assistant']);
|
||||
assert.match(stored[1]?.content ?? '', /\[big\]/);
|
||||
});
|
||||
|
||||
it('keeps sessions isolated', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, { model: 'fast', session: 'a', messages: [say('secret for A')] });
|
||||
await chat(h.base, { model: 'fast', session: 'b', messages: [say('secret for B')] });
|
||||
|
||||
const calls = h.big.callsTo('/chat/completions');
|
||||
const forB = calls[calls.length - 1]?.body.messages ?? [];
|
||||
assert.ok(!forB.some((m) => m.content.includes('secret for A')), 'session A must not leak into session B');
|
||||
});
|
||||
|
||||
it('behaves like a plain OpenAI endpoint with no session id', async () => {
|
||||
const h = await harness();
|
||||
const res = await chat(h.base, { model: 'fast', messages: [say('stateless please')] });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-session'), null);
|
||||
assert.equal(h.context.list().length, 0, 'nothing was stored');
|
||||
});
|
||||
});
|
||||
|
||||
describe('sessions — surviving a failover to a smaller window', () => {
|
||||
it('compacts the history to fit the model that actually serves it', async () => {
|
||||
const h = await harness({ context: { enabled: true, maxTurns: 4, reserveOutputTokens: 512, compactModel: 'fast', autoCompactAt: 0.8 } });
|
||||
|
||||
// Build a long thread on the 128k model.
|
||||
const long = 'This is a reasonably long turn about the project. '.repeat(120);
|
||||
for (let i = 0; i < 12; i++) {
|
||||
await chat(h.base, { model: 'fast', session: 'long', messages: [say(`turn ${i}: ${long}`)] });
|
||||
}
|
||||
|
||||
const beforeCalls = h.big.callsTo('/chat/completions').length;
|
||||
const sentToBig = h.big.callsTo('/chat/completions')[beforeCalls - 1]?.body.messages ?? [];
|
||||
assert.ok(sentToBig.length > 5, 'the big model received the full history');
|
||||
|
||||
// The big provider runs out; everything now has to go to the 8k model.
|
||||
h.big.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const res = await chat(h.base, { model: 'fast', session: 'long', messages: [say('given all that, what should I do?')] });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'small', 'it failed over');
|
||||
|
||||
const note = res.headers.get('x-freeway-context');
|
||||
assert.ok(note, 'the refit is reported on the response');
|
||||
assert.match(note, /refit=(window|compact|hard-truncate|drop-tool-noise)/);
|
||||
assert.match(note, /tokens=\d+->\d+/);
|
||||
|
||||
// The reshaped history genuinely fits the smaller window.
|
||||
const smallCalls = h.small.callsTo('/chat/completions');
|
||||
const sentToSmall = smallCalls[smallCalls.length - 1]?.body.messages ?? [];
|
||||
const chars = sentToSmall.reduce((n, m) => n + String(m.content).length, 0);
|
||||
assert.ok(chars / 4 < 8192, `history must fit 8k, estimated ${Math.round(chars / 4)} tokens`);
|
||||
|
||||
// And the newest question survived intact — that is the request itself.
|
||||
assert.ok(sentToSmall.some((m) => m.content.includes('given all that, what should I do?')));
|
||||
});
|
||||
|
||||
it('reports a refit even without a session, so plain failover still fits', async () => {
|
||||
// A client that manages its own history still benefits: the array it sent
|
||||
// is reshaped for whichever model ends up serving the call.
|
||||
const h = await harness();
|
||||
h.big.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const turn = 'padding '.repeat(300); // ~600 tokens each
|
||||
const messages = [{ role: 'system', content: 'be helpful' }];
|
||||
for (let i = 0; i < 20; i++) {
|
||||
messages.push({ role: i % 2 === 0 ? 'user' : 'assistant', content: `${i}: ${turn}` });
|
||||
}
|
||||
messages.push(say('and finally, a short question'));
|
||||
|
||||
const res = await chat(h.base, { model: 'fast', messages });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'small');
|
||||
const note = res.headers.get('x-freeway-context');
|
||||
assert.ok(note, 'stateless requests are refit too');
|
||||
assert.ok(!note.includes('refit=passthrough'), `it genuinely had to reshape, got "${note}"`);
|
||||
|
||||
const smallCalls = h.small.callsTo('/chat/completions');
|
||||
const sent = smallCalls[smallCalls.length - 1]?.body.messages ?? [];
|
||||
const chars = sent.reduce((n, m) => n + String(m.content).length, 0);
|
||||
assert.ok(chars / 4 < 8192, `must fit the 8k window, estimated ${Math.round(chars / 4)} tokens`);
|
||||
assert.ok(sent.some((m) => m.content.includes('and finally, a short question')), 'the actual request survived');
|
||||
});
|
||||
|
||||
it('returns 413 when the newest message alone cannot fit', async () => {
|
||||
const h = await harness();
|
||||
// Only the 8k model is available, and the message needs far more than that.
|
||||
h.big.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const huge = 'x'.repeat(200_000);
|
||||
const res = await chat(h.base, { model: 'fast', messages: [say(huge)] });
|
||||
|
||||
assert.equal(res.status, 413);
|
||||
const body = (await res.json()) as { error: { code: string }; freeway: { required: number; available: number } };
|
||||
assert.equal(body.error.code, 'context_too_large');
|
||||
assert.ok(body.freeway.required > body.freeway.available);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sessions — API', () => {
|
||||
it('lists and inspects sessions, then deletes one', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, { model: 'fast', session: 'inspect-me', messages: [say('hello there')] });
|
||||
|
||||
const list = (await (await fetch(`${h.base}/api/sessions`)).json()) as { sessions: { id: string; messages: number }[] };
|
||||
assert.ok(list.sessions.some((s) => s.id === 'inspect-me'));
|
||||
|
||||
const one = (await (await fetch(`${h.base}/api/sessions/inspect-me`)).json()) as { messages: { role: string }[] };
|
||||
assert.deepEqual(one.messages.map((m) => m.role), ['user', 'assistant']);
|
||||
|
||||
const del = await fetch(`${h.base}/api/sessions/inspect-me`, { method: 'DELETE' });
|
||||
assert.equal(del.status, 200);
|
||||
assert.equal(h.context.info('inspect-me'), null);
|
||||
});
|
||||
|
||||
it('404s an unknown session', async () => {
|
||||
const h = await harness();
|
||||
assert.equal((await fetch(`${h.base}/api/sessions/nope`)).status, 404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "./src",
|
||||
"outDir": "./dist",
|
||||
"tsBuildInfoFile": "./dist/.tsbuildinfo"
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"references": [{ "path": "../core" }]
|
||||
}
|
||||
Reference in new issue
Block a user