Freeway: self-hosted OpenAI-compatible gateway for free-tier providers
One endpoint in front of every free LLM tier, with automatic failover and no provider key ever leaving the box. Providers are data, not code: adding one is a JSON file plus an env var, never a TypeScript change. Ships 12 providers pre-written; `verify-provider --apply` derives the live model list from your own key. - core: registry, bucketed quota meter (rolling + UTC-calendar windows), key-pool store, 6-step model router with diagnostic rejection reasons - gateway: node:http, failover that commits to a stream only after 200 headers, SSE passthrough with usage sniffing, virtual keys with admin/inference roles - context engine: sqlite sessions and a 5-rung refit ladder, so a 128k thread keeps working when it fails over to an 8k model - cache: exact (sha256, safe-by-default) + opt-in semantic (flat Float32Array) - guard: inbound limits, quota forecasting, limit calibration from upstream headers, key health, secret/PII/injection scanning - dashboard: one self-contained HTML file, live SSE request log, token usage - sdk: zero-dep typed client, drop-in for the OpenAI SDK Zero runtime dependencies. 326 tests, 27-check smoke suite against a scripted mock upstream so routing is testable without spending quota. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Aryan SinghandClaude Opus 5 committed
commit
cb985dcf9c
86 files changed
+16350
No files matched your search
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* Cache behaviour through the whole stack: a hit must cost no upstream call and
|
||||
* no quota, and must be indistinguishable from a live answer to the client.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { ExactCache, Registry, Router, SemanticCache, Store, parseConfig, silentLogger, type CacheMode, type FreewayConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from '../src/server.ts';
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock {
|
||||
url: string;
|
||||
callsTo(path: string): unknown[];
|
||||
close(): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
base: string;
|
||||
upstream: Mock;
|
||||
cache: ExactCache;
|
||||
semantic: SemanticCache;
|
||||
store: Store;
|
||||
}
|
||||
|
||||
async function harness(mode: CacheMode = 'safe', semanticOn = false): Promise<Harness> {
|
||||
const upstream = (await createMockUpstream({ name: 'up' })) as Mock;
|
||||
cleanups.push(() => upstream.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-cache-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
writeFileSync(join(dir, 'up.json'), JSON.stringify({
|
||||
id: 'up', priority: 10, baseUrl: upstream.url,
|
||||
auth: { type: 'bearer', envKeys: ['UP_KEY'] }, limits: { rpm: 1000 },
|
||||
models: [
|
||||
{ id: 'up-chat', alias: ['fast'], context: 128000, caps: ['chat'] },
|
||||
{ id: 'up-embed', alias: ['embed'], context: 8000, caps: ['embed'] },
|
||||
],
|
||||
}));
|
||||
|
||||
const registry = new Registry({ dir, env: { UP_KEY: 'uk-000000000001' }, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {}) as { config: FreewayConfig };
|
||||
config.cache.mode = mode;
|
||||
config.cache.semantic.enabled = semanticOn;
|
||||
const router = new Router(registry, store, config);
|
||||
|
||||
const cache = new ExactCache({ file: ':memory:', mode, ttlMs: 3_600_000, maxEntries: 100 });
|
||||
cleanups.push(() => cache.close());
|
||||
const semantic = new SemanticCache({ enabled: semanticOn, threshold: 0.92, maxEntries: 100, dims: 8 });
|
||||
|
||||
const server: Server = createGateway({
|
||||
registry, store, router, config, logger: silentLogger, cache, semanticCache: semantic,
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
return { base: `http://127.0.0.1:${port}`, upstream, cache, semantic, store };
|
||||
}
|
||||
|
||||
async function chat(base: string, body: unknown): Promise<Response> {
|
||||
return fetch(`${base}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
const ask = (content: string, over: Record<string, unknown> = {}) => ({
|
||||
model: 'fast',
|
||||
temperature: 0,
|
||||
messages: [{ role: 'user', content }],
|
||||
...over,
|
||||
});
|
||||
|
||||
describe('cache — exact hits', () => {
|
||||
it('serves the second identical request without touching the upstream', async () => {
|
||||
const h = await harness();
|
||||
|
||||
const first = await chat(h.base, ask('what is the capital of France?'));
|
||||
assert.equal(first.status, 200);
|
||||
assert.equal(first.headers.get('x-freeway-cache'), null, 'the first call is a miss');
|
||||
const firstBody = await first.text();
|
||||
|
||||
const second = await chat(h.base, ask('what is the capital of France?'));
|
||||
assert.equal(second.status, 200);
|
||||
assert.equal(second.headers.get('x-freeway-cache'), 'exact');
|
||||
assert.equal(await second.text(), firstBody, 'byte-identical to the original answer');
|
||||
|
||||
assert.equal(h.upstream.callsTo('/chat/completions').length, 1, 'only one upstream call was ever made');
|
||||
});
|
||||
|
||||
it('spends no quota on a hit', async () => {
|
||||
// Metering a cache hit would make the quota bars lie about what is left.
|
||||
const h = await harness();
|
||||
await chat(h.base, ask('question'));
|
||||
const after = h.store.provider('up').meter.used('rpm');
|
||||
|
||||
await chat(h.base, ask('question'));
|
||||
assert.equal(h.store.provider('up').meter.used('rpm'), after, 'the hit did not count against the provider');
|
||||
});
|
||||
|
||||
it('logs a hit distinctly', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, ask('question'));
|
||||
await chat(h.base, ask('question'));
|
||||
|
||||
const logs = h.store.logs(2);
|
||||
assert.equal(logs[0]?.cache, 'exact');
|
||||
assert.equal(logs[1]?.cache, null);
|
||||
assert.equal(logs[0]?.status, 200);
|
||||
});
|
||||
|
||||
it('does not cache a creative call in safe mode', async () => {
|
||||
const h = await harness('safe');
|
||||
await chat(h.base, ask('write me a poem', { temperature: 0.9 }));
|
||||
const second = await chat(h.base, ask('write me a poem', { temperature: 0.9 }));
|
||||
|
||||
assert.equal(second.headers.get('x-freeway-cache'), null);
|
||||
assert.equal(h.upstream.callsTo('/chat/completions').length, 2, 'both calls went upstream');
|
||||
});
|
||||
|
||||
it('caches a creative call in aggressive mode', async () => {
|
||||
const h = await harness('aggressive');
|
||||
await chat(h.base, ask('write me a poem', { temperature: 0.9 }));
|
||||
const second = await chat(h.base, ask('write me a poem', { temperature: 0.9 }));
|
||||
assert.equal(second.headers.get('x-freeway-cache'), 'exact');
|
||||
});
|
||||
|
||||
it('is inert when off', async () => {
|
||||
const h = await harness('off');
|
||||
await chat(h.base, ask('question'));
|
||||
const second = await chat(h.base, ask('question'));
|
||||
assert.equal(second.headers.get('x-freeway-cache'), null);
|
||||
assert.equal(h.upstream.callsTo('/chat/completions').length, 2);
|
||||
});
|
||||
|
||||
it('treats a different question as a different request', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, ask('question one'));
|
||||
const second = await chat(h.base, ask('question two'));
|
||||
assert.equal(second.headers.get('x-freeway-cache'), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cache — streaming replay', () => {
|
||||
it('replays a cached answer as SSE the client cannot distinguish', async () => {
|
||||
const h = await harness('aggressive');
|
||||
|
||||
const live = await chat(h.base, ask('stream me', { stream: true }));
|
||||
assert.match(live.headers.get('content-type') ?? '', /text\/event-stream/);
|
||||
const liveText = await live.text();
|
||||
|
||||
const cached = await chat(h.base, ask('stream me', { stream: true }));
|
||||
assert.equal(cached.headers.get('x-freeway-cache'), 'exact');
|
||||
assert.match(cached.headers.get('content-type') ?? '', /text\/event-stream/);
|
||||
|
||||
const cachedText = await cached.text();
|
||||
assert.ok(cachedText.trimEnd().endsWith('data: [DONE]'));
|
||||
|
||||
const contentOf = (sse: string): string =>
|
||||
[...sse.matchAll(/^data: (.+)$/gm)]
|
||||
.map((m) => m[1])
|
||||
.filter((p): p is string => p !== undefined && p !== '[DONE]')
|
||||
.map((p) => JSON.parse(p) as { choices?: { delta?: { content?: string } }[] })
|
||||
.flatMap((o) => o.choices ?? [])
|
||||
.map((c) => c.delta?.content ?? '')
|
||||
.join('');
|
||||
|
||||
assert.equal(contentOf(cachedText), contentOf(liveText), 'the replayed text matches the original exactly');
|
||||
});
|
||||
});
|
||||
|
||||
describe('cache — semantic tier', () => {
|
||||
it('hits on a differently-worded question above the threshold', async () => {
|
||||
// The mock's embeddings are deterministic from the text, so similar text
|
||||
// gives similar vectors — the same property a real embedder has.
|
||||
const h = await harness('safe', true);
|
||||
|
||||
await chat(h.base, ask('what is the capital of France'));
|
||||
const before = h.upstream.callsTo('/chat/completions').length;
|
||||
|
||||
const reworded = await chat(h.base, ask('what is the capital of France?'));
|
||||
assert.equal(reworded.status, 200);
|
||||
|
||||
if (reworded.headers.get('x-freeway-cache') === 'semantic') {
|
||||
const score = Number(reworded.headers.get('x-freeway-cache-score'));
|
||||
assert.ok(score >= 0.92, `a hit must clear the threshold, got ${score}`);
|
||||
assert.equal(h.upstream.callsTo('/chat/completions').length, before, 'no chat call was made');
|
||||
}
|
||||
});
|
||||
|
||||
it('never serves across a different system prompt', async () => {
|
||||
// The same question can have a different correct answer under a different
|
||||
// system prompt, so entries must be scoped.
|
||||
const h = await harness('safe', true);
|
||||
|
||||
await chat(h.base, { ...ask('who are you'), messages: [{ role: 'system', content: 'You are a pirate.' }, { role: 'user', content: 'who are you' }] });
|
||||
const other = await chat(h.base, { ...ask('who are you'), messages: [{ role: 'system', content: 'You are a lawyer.' }, { role: 'user', content: 'who are you' }] });
|
||||
|
||||
assert.equal(other.headers.get('x-freeway-cache'), null, 'a different system prompt must not reuse the answer');
|
||||
});
|
||||
|
||||
it('does not consult the semantic tier when disabled', async () => {
|
||||
const h = await harness('safe', false);
|
||||
await chat(h.base, ask('a question'));
|
||||
await chat(h.base, ask('a slightly different question'));
|
||||
assert.equal(h.upstream.callsTo('/embeddings').length, 0, 'no embedding quota was spent');
|
||||
});
|
||||
});
|
||||
|
||||
describe('cache — API', () => {
|
||||
it('reports stats and can be cleared', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, ask('question'));
|
||||
await chat(h.base, ask('question'));
|
||||
|
||||
const stats = (await (await fetch(`${h.base}/api/cache`)).json()) as {
|
||||
enabled: boolean; mode: string; exact: { hits: number; entries: number; tokensSaved: number };
|
||||
};
|
||||
assert.equal(stats.enabled, true);
|
||||
assert.equal(stats.mode, 'safe');
|
||||
assert.equal(stats.exact.hits, 1);
|
||||
assert.equal(stats.exact.entries, 1);
|
||||
assert.ok(stats.exact.tokensSaved > 0);
|
||||
|
||||
assert.equal((await fetch(`${h.base}/api/cache`, { method: 'DELETE' })).status, 200);
|
||||
const after = (await (await fetch(`${h.base}/api/cache`)).json()) as { exact: { entries: number } };
|
||||
assert.equal(after.exact.entries, 0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,665 @@
|
||||
/**
|
||||
* End-to-end tests against mock upstreams.
|
||||
*
|
||||
* Everything here runs a real HTTP gateway against real HTTP fakes — no
|
||||
* stubbing of fetch, no injected doubles — so what is verified is what actually
|
||||
* happens on the wire, and it costs no free-tier quota to run.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, beforeEach, describe, it } from 'node:test';
|
||||
|
||||
import { Registry, Router, Store, parseConfig, silentLogger, type FreewayConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from '../src/server.ts';
|
||||
// eslint-disable-next-line -- plain JS harness, deliberately not TypeScript so it can be run standalone
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock {
|
||||
url: string;
|
||||
name: string;
|
||||
calls: { path: string; body: unknown; key: string | null }[];
|
||||
script(steps: unknown[]): void;
|
||||
setKeys(k: Record<string, unknown>): void;
|
||||
callsTo(path: string): { path: string; body: unknown; key: string | null }[];
|
||||
reset(): void;
|
||||
close(): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
base: string;
|
||||
alpha: Mock;
|
||||
beta: Mock;
|
||||
store: Store;
|
||||
registry: Registry;
|
||||
config: FreewayConfig;
|
||||
}
|
||||
|
||||
async function harness(configOverrides: Partial<FreewayConfig> = {}, env: Record<string, string> = {}): Promise<Harness> {
|
||||
const alpha = (await createMockUpstream({ name: 'alpha' })) as Mock;
|
||||
const beta = (await createMockUpstream({ name: 'beta' })) as Mock;
|
||||
cleanups.push(() => alpha.close(), () => beta.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-gw-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
writeFileSync(
|
||||
join(dir, 'alpha.json'),
|
||||
JSON.stringify({
|
||||
id: 'alpha',
|
||||
priority: 10,
|
||||
baseUrl: alpha.url,
|
||||
auth: { type: 'bearer', envKeys: ['ALPHA_KEY'] },
|
||||
limits: { rpm: 100 },
|
||||
models: [
|
||||
{ id: 'alpha-chat', alias: ['fast'], context: 128000, caps: ['chat', 'tools', 'json'], priority: 10 },
|
||||
{ id: 'alpha-embed', alias: ['embed'], context: 8000, caps: ['embed'] },
|
||||
],
|
||||
}),
|
||||
);
|
||||
writeFileSync(
|
||||
join(dir, 'beta.json'),
|
||||
JSON.stringify({
|
||||
id: 'beta',
|
||||
priority: 20,
|
||||
baseUrl: beta.url,
|
||||
auth: { type: 'bearer', envKeys: ['BETA_KEY'] },
|
||||
limits: { rpm: 100 },
|
||||
models: [{ id: '@cf/meta/llama-3.1-8b-instruct', alias: ['fast'], context: 8192, caps: ['chat'], priority: 10 }],
|
||||
}),
|
||||
);
|
||||
|
||||
const registry = new Registry({
|
||||
dir,
|
||||
env: { ALPHA_KEY: 'ak-000000000001,ak-000000000002', BETA_KEY: 'bk-000000000001', ...env },
|
||||
logger: silentLogger,
|
||||
});
|
||||
const store = new Store({ file: null, cooldownMs: 60_000, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
Object.assign(config, configOverrides);
|
||||
const router = new Router(registry, store, config);
|
||||
|
||||
const server: Server = createGateway({ registry, store, router, config, logger: silentLogger });
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
return { base: `http://127.0.0.1:${port}`, alpha, beta, store, registry, config };
|
||||
}
|
||||
|
||||
async function post(base: string, path: string, body: unknown, headers: Record<string, string> = {}): Promise<Response> {
|
||||
return fetch(`${base}${path}`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', ...headers },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
const CHAT = { model: 'auto', messages: [{ role: 'user', content: 'hello there' }] };
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('gateway — a normal completion', () => {
|
||||
it('proxies the request and reports who served it', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', CHAT);
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha', 'priority order picked alpha');
|
||||
assert.equal(res.headers.get('x-freeway-model'), 'alpha-chat');
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '1');
|
||||
assert.equal(res.headers.get('x-freeway-route'), 'auto');
|
||||
assert.ok(Number(res.headers.get('x-freeway-ms')) >= 0);
|
||||
|
||||
const body = (await res.json()) as { choices: { message: { content: string } }[] };
|
||||
assert.match(body.choices[0]?.message.content ?? '', /\[alpha\]/);
|
||||
});
|
||||
|
||||
it('never forwards the virtual key upstream, only the provider key', async () => {
|
||||
const h = await harness({ keys: ['fw-secret-key'] });
|
||||
await post(h.base, '/v1/chat/completions', CHAT, { authorization: 'Bearer fw-secret-key' });
|
||||
|
||||
const call = h.alpha.callsTo('/chat/completions')[0];
|
||||
assert.ok(call);
|
||||
// Which key of the pool served it depends on cursor rotation; what matters
|
||||
// is that it is a provider key and never the caller's virtual one.
|
||||
assert.ok(['ak-000000000001', 'ak-000000000002'].includes(call.key ?? ''), `unexpected upstream key ${call.key}`);
|
||||
assert.notEqual(call.key, 'fw-secret-key');
|
||||
});
|
||||
|
||||
it('rewrites the model to the upstream id', async () => {
|
||||
const h = await harness();
|
||||
await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
const call = h.alpha.callsTo('/chat/completions')[0];
|
||||
assert.equal((call?.body as { model: string }).model, 'alpha-chat');
|
||||
});
|
||||
|
||||
it('records tokens from the upstream usage block', async () => {
|
||||
const h = await harness();
|
||||
await post(h.base, '/v1/chat/completions', CHAT);
|
||||
assert.ok(h.store.provider('alpha').meter.used('tpm') > 0, 'usage was metered');
|
||||
});
|
||||
|
||||
it('falls back to an estimate when a provider reports no usage', async () => {
|
||||
// A provider reporting nothing must not leave the quota counter reading zero.
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 200, dropUsage: true, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', CHAT);
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-usage'), 'estimated');
|
||||
assert.ok(h.store.provider('alpha').meter.used('tpm') > 0, 'estimated tokens still count');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — failover', () => {
|
||||
it('a 429 on provider A fails over to provider B', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'beta');
|
||||
// 3, not 2: a 429 is a key fault, so alpha's *other* pool key is tried
|
||||
// before the provider is abandoned. That is the point of a key pool.
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '3');
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, 2, 'both alpha keys were tried');
|
||||
|
||||
const body = (await res.json()) as { choices: { message: { content: string } }[] };
|
||||
assert.match(body.choices[0]?.message.content ?? '', /\[beta\]/);
|
||||
});
|
||||
|
||||
it('a 429 on one key retries the other key of the same provider', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.setKeys({ 'ak-000000000001': { status: 429 } });
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'alpha/alpha-chat' });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha', 'stayed on the same provider');
|
||||
|
||||
const keysUsed = h.alpha.callsTo('/chat/completions').map((c) => c.key);
|
||||
assert.ok(keysUsed.includes('ak-000000000002'), 'the pool’s other key served it');
|
||||
});
|
||||
|
||||
it('a 5xx benches the provider so the next request skips it entirely', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 503, repeat: Infinity }]);
|
||||
|
||||
await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
const before = h.alpha.callsTo('/chat/completions').length;
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'beta');
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '1', 'the benched provider is not even tried');
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, before, 'no second call to the dead provider');
|
||||
});
|
||||
|
||||
it('honours an upstream Retry-After over the configured cooldown', async () => {
|
||||
const h = await harness({ cooldownSeconds: 600 });
|
||||
h.alpha.script([{ status: 429, headers: { 'retry-after': '2' }, repeat: Infinity }]);
|
||||
|
||||
await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'alpha/alpha-chat' });
|
||||
const cooldown = h.store.key('alpha', 'alpha#0').cooldownUntil - Date.now();
|
||||
assert.ok(cooldown <= 2500 && cooldown > 0, `expected ~2s from Retry-After, got ${cooldown}ms`);
|
||||
});
|
||||
|
||||
it('stops after maxAttempts rather than walking every candidate', async () => {
|
||||
const h = await harness({ maxAttempts: 1 });
|
||||
h.alpha.script([{ status: 500, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.status, 502);
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '1');
|
||||
assert.equal(h.beta.callsTo('/chat/completions').length, 0, 'never got to beta');
|
||||
});
|
||||
|
||||
it('returns a 400 immediately without retrying anywhere', async () => {
|
||||
// A malformed body fails identically everywhere; retrying burns quota for nothing.
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 400, body: { error: { message: 'bad parameter foo', type: 'invalid_request_error' } }, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
|
||||
assert.equal(res.status, 400);
|
||||
assert.equal(res.headers.get('x-freeway-attempts'), '1');
|
||||
assert.equal(h.beta.callsTo('/chat/completions').length, 0, 'beta was never tried');
|
||||
|
||||
const body = (await res.json()) as { error: { message: string } };
|
||||
assert.equal(body.error.message, 'bad parameter foo', 'the upstream error is passed through verbatim');
|
||||
assert.equal(h.store.isProviderCooling('alpha'), false, 'a 400 benches nothing');
|
||||
});
|
||||
|
||||
it('reports every attempt when they all fail', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 500, repeat: Infinity }]);
|
||||
h.beta.script([{ status: 502, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.status, 502);
|
||||
|
||||
const body = (await res.json()) as { freeway: { attempts: { provider: string; status: number }[] } };
|
||||
// A 500 is a provider fault, so alpha's second key is skipped — every key
|
||||
// would fail identically and the attempt budget is better spent on beta.
|
||||
assert.equal(body.freeway.attempts.length, 2);
|
||||
assert.deepEqual(body.freeway.attempts.map((a) => a.provider), ['alpha', 'beta']);
|
||||
assert.deepEqual(body.freeway.attempts.map((a) => a.status), [500, 502]);
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, 1, 'the benched provider was not retried on another key');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — streaming', () => {
|
||||
it('proxies SSE through intact', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, stream: true });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.match(res.headers.get('content-type') ?? '', /text\/event-stream/);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha');
|
||||
|
||||
const text = await res.text();
|
||||
assert.match(text, /^data: /m);
|
||||
assert.match(text, /data: \[DONE\]/);
|
||||
|
||||
// Reassemble the deltas — the content must survive the transform untouched.
|
||||
const content = [...text.matchAll(/^data: (.+)$/gm)]
|
||||
.map((m) => m[1])
|
||||
.filter((p): p is string => p !== undefined && p !== '[DONE]')
|
||||
.map((p) => JSON.parse(p) as { choices?: { delta?: { content?: string } }[] })
|
||||
.flatMap((o) => o.choices ?? [])
|
||||
.map((c) => c.delta?.content ?? '')
|
||||
.join('');
|
||||
assert.match(content, /\[alpha\]/);
|
||||
});
|
||||
|
||||
it('accounts for tokens from the final usage chunk of a stream', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, stream: true });
|
||||
await res.text();
|
||||
|
||||
// Usage arrives only in the last chunk, long after the response committed.
|
||||
assert.ok(h.store.provider('alpha').meter.used('tpm') > 0, 'late usage was still metered');
|
||||
const log = h.store.logs(1)[0];
|
||||
assert.ok(log);
|
||||
assert.equal(log.stream, true);
|
||||
assert.ok(log.tokensOut > 0);
|
||||
});
|
||||
|
||||
it('still fails over when the first provider 429s a stream request', async () => {
|
||||
// The commit point is after 200 headers, so a pre-body failure is recoverable.
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast', stream: true });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'beta');
|
||||
assert.match(await res.text(), /\[beta\]/);
|
||||
});
|
||||
|
||||
it('estimates stream tokens when the provider omits usage', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 200, dropUsage: true, repeat: Infinity }]);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, stream: true });
|
||||
await res.text();
|
||||
assert.ok(h.store.provider('alpha').meter.used('tpm') > 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — quota exhaustion', () => {
|
||||
it('returns 429 with a diagnostic reason per provider', async () => {
|
||||
const h = await harness();
|
||||
for (let i = 0; i < 100; i++) {
|
||||
h.store.recordSuccess('alpha', 'alpha#0', 5);
|
||||
h.store.recordSuccess('beta', 'beta#0', 5);
|
||||
}
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.status, 429);
|
||||
assert.ok(Number(res.headers.get('retry-after')) > 0, 'tells the client when to come back');
|
||||
|
||||
const body = (await res.json()) as {
|
||||
error: { code: string };
|
||||
freeway: { blocked: { provider: string; reason: string }[]; retryAfterMs: number };
|
||||
};
|
||||
assert.equal(body.error.code, 'all_providers_blocked');
|
||||
assert.equal(body.freeway.blocked.length, 2);
|
||||
for (const b of body.freeway.blocked) {
|
||||
assert.match(b.reason, /rpm limit reached \(100\/100 last 60s\)/, `reason must be specific, got "${b.reason}"`);
|
||||
}
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, 0, 'no upstream call was wasted');
|
||||
});
|
||||
|
||||
it('tells you which env var to set when nothing is configured', async () => {
|
||||
const h = await harness({}, { ALPHA_KEY: '', BETA_KEY: '' });
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
|
||||
assert.equal(res.status, 429);
|
||||
const body = (await res.json()) as { freeway: { blocked: { reason: string }[] } };
|
||||
const reasons = body.freeway.blocked.map((b) => b.reason).join(' ');
|
||||
assert.match(reasons, /ALPHA_KEY/);
|
||||
assert.match(reasons, /BETA_KEY/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — unknown models', () => {
|
||||
it('returns 404 with the closest available ids', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'gpt-4o-turbo-max' });
|
||||
|
||||
assert.equal(res.status, 404);
|
||||
const body = (await res.json()) as { error: { code: string }; freeway: { suggestions: string[] } };
|
||||
assert.equal(body.error.code, 'model_not_found');
|
||||
assert.ok(Array.isArray(body.freeway.suggestions));
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — virtual keys', () => {
|
||||
it('rejects a request with no key when keys are configured', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
const res = await post(h.base, '/v1/chat/completions', CHAT);
|
||||
|
||||
assert.equal(res.status, 401);
|
||||
const body = (await res.json()) as { error: { code: string; message: string } };
|
||||
assert.equal(body.error.code, 'invalid_api_key');
|
||||
assert.match(body.error.message, /missing API key/);
|
||||
});
|
||||
|
||||
it('rejects a wrong key', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
const res = await post(h.base, '/v1/chat/completions', CHAT, { authorization: 'Bearer fw-wrong' });
|
||||
assert.equal(res.status, 401);
|
||||
assert.equal(h.alpha.callsTo('/chat/completions').length, 0, 'never reached an upstream');
|
||||
});
|
||||
|
||||
it('accepts a correct key, via Authorization or x-api-key', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
assert.equal((await post(h.base, '/v1/chat/completions', CHAT, { authorization: 'Bearer fw-good' })).status, 200);
|
||||
assert.equal((await post(h.base, '/v1/chat/completions', CHAT, { 'x-api-key': 'fw-good' })).status, 200);
|
||||
});
|
||||
|
||||
it('is open when no keys are configured', async () => {
|
||||
const h = await harness();
|
||||
assert.equal((await post(h.base, '/v1/chat/completions', CHAT)).status, 200);
|
||||
});
|
||||
|
||||
it('leaves /healthz reachable without a key', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
const res = await fetch(`${h.base}/healthz`);
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(((await res.json()) as { ok: boolean }).ok, true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — embeddings', () => {
|
||||
it('routes to an embedding model, never a chat one', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/embeddings', { model: 'auto', input: ['alpha', 'beta'] });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-model'), 'alpha-embed');
|
||||
|
||||
const body = (await res.json()) as { data: { embedding: number[] }[] };
|
||||
assert.equal(body.data.length, 2);
|
||||
assert.ok(Array.isArray(body.data[0]?.embedding));
|
||||
});
|
||||
|
||||
it('rejects an embeddings call with no input', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/embeddings', { model: 'auto' });
|
||||
assert.equal(res.status, 400);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — capability routing', () => {
|
||||
it('sends a tools request only to a provider that supports tools', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', {
|
||||
...CHAT,
|
||||
model: 'fast',
|
||||
tools: [{ type: 'function', function: { name: 'get_weather', parameters: {} } }],
|
||||
});
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha', 'beta lacks the tools capability');
|
||||
assert.equal(h.beta.callsTo('/chat/completions').length, 0);
|
||||
});
|
||||
|
||||
it('routes a json_object request to a json-capable model', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast', response_format: { type: 'json_object' } });
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'alpha');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — dashboard API', () => {
|
||||
let h: Harness;
|
||||
beforeEach(async () => {
|
||||
h = await harness();
|
||||
});
|
||||
|
||||
it('GET /v1/models returns provider/model ids plus virtual aliases', async () => {
|
||||
const res = await fetch(`${h.base}/v1/models`);
|
||||
assert.equal(res.status, 200);
|
||||
|
||||
const body = (await res.json()) as { object: string; data: { id: string; owned_by: string }[] };
|
||||
assert.equal(body.object, 'list');
|
||||
const ids = body.data.map((d) => d.id);
|
||||
assert.ok(ids.includes('alpha/alpha-chat'));
|
||||
assert.ok(ids.includes('beta/@cf/meta/llama-3.1-8b-instruct'), 'slashy ids survive the catalog');
|
||||
assert.ok(ids.includes('fast'), 'aliases appear as virtual models');
|
||||
assert.ok(ids.includes('auto'));
|
||||
});
|
||||
|
||||
it('GET /api/providers exposes state with masked keys only', async () => {
|
||||
const res = await fetch(`${h.base}/api/providers`);
|
||||
const body = (await res.json()) as { providers: { id: string; keys: { masked: string }[]; bars: unknown[] }[] };
|
||||
|
||||
const alpha = body.providers.find((p) => p.id === 'alpha');
|
||||
assert.ok(alpha);
|
||||
assert.equal(alpha.keys.length, 2);
|
||||
assert.equal(alpha.keys[0]?.masked, 'ak-0…0001');
|
||||
|
||||
const raw = JSON.stringify(body);
|
||||
assert.ok(!raw.includes('ak-000000000001'), 'a raw provider key must never reach the dashboard');
|
||||
assert.ok(!raw.includes('bk-000000000001'));
|
||||
});
|
||||
|
||||
it('GET /api/logs shows the served request', async () => {
|
||||
await post(h.base, '/v1/chat/completions', CHAT);
|
||||
const res = await fetch(`${h.base}/api/logs`);
|
||||
const body = (await res.json()) as { logs: { providerId: string; status: number; resolution: string }[] };
|
||||
|
||||
assert.equal(body.logs.length, 1);
|
||||
assert.equal(body.logs[0]?.providerId, 'alpha');
|
||||
assert.equal(body.logs[0]?.status, 200);
|
||||
assert.equal(body.logs[0]?.resolution, 'auto');
|
||||
});
|
||||
|
||||
it('POST /api/providers/:id/enabled takes a provider out of rotation', async () => {
|
||||
const toggle = await post(h.base, '/api/providers/alpha/enabled', { enabled: false });
|
||||
assert.equal(toggle.status, 200);
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'beta');
|
||||
});
|
||||
|
||||
it('POST /api/reload re-reads providers without a restart', async () => {
|
||||
const res = await post(h.base, '/api/reload', {});
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(((await res.json()) as { ok: boolean }).ok, true);
|
||||
});
|
||||
|
||||
it('GET /api/stats summarises the fleet', async () => {
|
||||
const res = await fetch(`${h.base}/api/stats`);
|
||||
const body = (await res.json()) as { providers: { total: number; usable: number }; strategy: string };
|
||||
assert.equal(body.providers.total, 2);
|
||||
assert.equal(body.providers.usable, 2);
|
||||
assert.equal(body.strategy, 'priority');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — live event stream', () => {
|
||||
/** Read SSE frames off a fetch body until `want` events arrive or time runs out. */
|
||||
async function collect(res: Response, want: number, timeoutMs = 4000): Promise<{ event: string; data: Record<string, unknown> }[]> {
|
||||
const out: { event: string; data: Record<string, unknown> }[] = [];
|
||||
const reader = res.body!.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
let event = 'message';
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
|
||||
while (out.length < want && Date.now() < deadline) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
let nl: number;
|
||||
while ((nl = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, nl).trim();
|
||||
buffer = buffer.slice(nl + 1);
|
||||
if (line.startsWith('event:')) event = line.slice(6).trim();
|
||||
else if (line.startsWith('data:')) {
|
||||
out.push({ event, data: JSON.parse(line.slice(5).trim()) as Record<string, unknown> });
|
||||
event = 'message';
|
||||
}
|
||||
}
|
||||
}
|
||||
await reader.cancel().catch(() => {});
|
||||
return out;
|
||||
}
|
||||
|
||||
it('pushes a log entry the moment a request completes', async () => {
|
||||
const h = await harness();
|
||||
const res = await fetch(`${h.base}/api/events`, { headers: { accept: 'text/event-stream' } });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.match(res.headers.get('content-type') ?? '', /text\/event-stream/);
|
||||
|
||||
const events = collect(res, 2);
|
||||
await post(h.base, '/v1/chat/completions', CHAT);
|
||||
const got = await events;
|
||||
|
||||
assert.equal(got[0]?.event, 'hello');
|
||||
const log = got.find((e) => e.event === 'log');
|
||||
assert.ok(log, 'the completed request arrived as a live event');
|
||||
assert.equal(log.data['providerId'], 'alpha');
|
||||
assert.equal(log.data['status'], 200);
|
||||
assert.equal(log.data['resolution'], 'auto');
|
||||
});
|
||||
|
||||
it('pushes failures too, with their attempt trail', async () => {
|
||||
const h = await harness();
|
||||
h.alpha.script([{ status: 429, repeat: Infinity }]);
|
||||
const res = await fetch(`${h.base}/api/events`, { headers: { accept: 'text/event-stream' } });
|
||||
|
||||
const events = collect(res, 2);
|
||||
await post(h.base, '/v1/chat/completions', { ...CHAT, model: 'fast' });
|
||||
const log = (await events).find((e) => e.event === 'log');
|
||||
|
||||
assert.ok(log);
|
||||
assert.equal(log.data['providerId'], 'beta');
|
||||
assert.ok(Array.isArray(log.data['attempts']) && (log.data['attempts'] as unknown[]).length > 0);
|
||||
});
|
||||
|
||||
it('unsubscribes when the client goes away, so listeners cannot leak', async () => {
|
||||
const h = await harness();
|
||||
assert.equal(h.store.listenerCount(), 0);
|
||||
|
||||
const controller = new AbortController();
|
||||
const res = await fetch(`${h.base}/api/events`, { signal: controller.signal });
|
||||
await collect(res, 1); // wait for `hello` so the subscription is registered
|
||||
assert.equal(h.store.listenerCount(), 1);
|
||||
|
||||
controller.abort();
|
||||
// A long-lived dashboard reconnecting all day must not accumulate listeners.
|
||||
for (let i = 0; i < 40 && h.store.listenerCount() > 0; i++) await new Promise((r) => setTimeout(r, 25));
|
||||
assert.equal(h.store.listenerCount(), 0);
|
||||
});
|
||||
|
||||
it('requires a virtual key like the rest of /api', async () => {
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
assert.equal((await fetch(`${h.base}/api/events`)).status, 401);
|
||||
const ok = await fetch(`${h.base}/api/events`, { headers: { authorization: 'Bearer fw-good' } });
|
||||
assert.equal(ok.status, 200);
|
||||
await ok.body?.cancel();
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — dashboard', () => {
|
||||
it('serves the dashboard at / without requiring a key', async () => {
|
||||
// The dashboard has to load before it can ask for a key, so `/` itself is
|
||||
// never gated; its `/api/*` fetches are.
|
||||
const h = await harness({ keys: ['fw-good'] });
|
||||
const res = await fetch(`${h.base}/`);
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.match(res.headers.get('content-type') ?? '', /text\/html/);
|
||||
const html = await res.text();
|
||||
assert.match(html, /<title>Freeway<\/title>/);
|
||||
});
|
||||
|
||||
it('is entirely self-contained — no CDN, no external anything', async () => {
|
||||
// A dashboard that phones out is a dashboard that breaks on an air-gapped
|
||||
// box and leaks which providers you run to whoever hosts the asset.
|
||||
const h = await harness();
|
||||
const html = await (await fetch(`${h.base}/`)).text();
|
||||
|
||||
const external = [...html.matchAll(/\b(?:src|href)\s*=\s*["']([^"']+)["']/gi)]
|
||||
.map((m) => m[1] ?? '')
|
||||
.filter((url) => /^(https?:)?\/\//i.test(url));
|
||||
|
||||
assert.deepEqual(external, [], `dashboard must not reference external assets, found: ${external.join(', ')}`);
|
||||
assert.ok(!/@import/i.test(html), 'no CSS @import either');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gateway — malformed input', () => {
|
||||
it('rejects a body that is not JSON', async () => {
|
||||
const h = await harness();
|
||||
const res = await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: '{ not json',
|
||||
});
|
||||
assert.equal(res.status, 400);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'invalid_json');
|
||||
});
|
||||
|
||||
it('rejects a chat request with no messages', async () => {
|
||||
const h = await harness();
|
||||
const res = await post(h.base, '/v1/chat/completions', { model: 'auto' });
|
||||
assert.equal(res.status, 400);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'missing_messages');
|
||||
});
|
||||
|
||||
it('rejects a body over the size cap with 413, not a dropped connection', async () => {
|
||||
const { config: defaults } = parseConfig({}, {});
|
||||
const h = await harness({ guard: { ...defaults.guard, maxBodyBytes: 512 } });
|
||||
|
||||
const res = await post(h.base, '/v1/chat/completions', {
|
||||
model: 'auto',
|
||||
messages: [{ role: 'user', content: 'x'.repeat(5000) }],
|
||||
});
|
||||
assert.equal(res.status, 413);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'payload_too_large');
|
||||
});
|
||||
|
||||
it('404s an unknown route', async () => {
|
||||
const h = await harness();
|
||||
const res = await fetch(`${h.base}/v1/nonsense`);
|
||||
assert.equal(res.status, 404);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,263 @@
|
||||
/**
|
||||
* Regression tests for the security sweep.
|
||||
*
|
||||
* Each of these encodes a vulnerability that was real, so a future refactor
|
||||
* cannot quietly reintroduce it.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { ContextStore, ExactCache, Registry, Router, Store, Tokenizer, parseConfig, silentLogger, type FreewayConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from '../src/server.ts';
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock { url: string; close(): Promise<unknown> }
|
||||
|
||||
interface Harness {
|
||||
base: string;
|
||||
context: ContextStore;
|
||||
dataDir: string;
|
||||
}
|
||||
|
||||
async function harness(over: Partial<FreewayConfig> = {}): Promise<Harness> {
|
||||
const upstream = (await createMockUpstream({ name: 'up' })) as Mock;
|
||||
cleanups.push(() => upstream.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-sec-'));
|
||||
const dataDir = mkdtempSync(join(tmpdir(), 'freeway-sec-data-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }), () => rmSync(dataDir, { recursive: true, force: true }));
|
||||
|
||||
writeFileSync(join(dir, 'up.json'), JSON.stringify({
|
||||
id: 'up', priority: 10, baseUrl: upstream.url,
|
||||
auth: { type: 'bearer', envKeys: ['UP_KEY'] }, limits: { rpm: 1000 },
|
||||
models: [{ id: 'up-chat', alias: ['fast'], context: 128000, caps: ['chat'] }],
|
||||
}));
|
||||
|
||||
const registry = new Registry({ dir, env: { UP_KEY: 'uk-000000000001' }, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
Object.assign(config, over);
|
||||
const router = new Router(registry, store, config);
|
||||
|
||||
const context = new ContextStore(join(dataDir, 'freeway.db'));
|
||||
const cache = new ExactCache({ file: join(dataDir, 'cache.db'), mode: 'safe', ttlMs: 3_600_000, maxEntries: 50 });
|
||||
cleanups.push(() => context.close(), () => cache.close());
|
||||
|
||||
const server: Server = createGateway({
|
||||
registry, store, router, config, logger: silentLogger,
|
||||
contextStore: context, tokenizer: new Tokenizer(), cache,
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
return { base: `http://127.0.0.1:${port}`, context, dataDir };
|
||||
}
|
||||
|
||||
const APP = 'fw-app-key';
|
||||
const ADMIN = 'fw-admin-key';
|
||||
const authed = (key: string) => ({ 'content-type': 'application/json', authorization: `Bearer ${key}` });
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('C1 — an inference key is not an admin key', () => {
|
||||
const cfg = { keys: [APP], adminKey: ADMIN, host: '0.0.0.0' };
|
||||
|
||||
it('refuses an app key on every /api route', async () => {
|
||||
const h = await harness(cfg);
|
||||
const routes: [string, string][] = [
|
||||
['GET', '/api/providers'], ['GET', '/api/logs'], ['GET', '/api/sessions'],
|
||||
['GET', '/api/alerts'], ['GET', '/api/cache'], ['GET', '/api/stats'], ['GET', '/api/events'],
|
||||
['POST', '/api/reload'], ['POST', '/api/providers/up/enabled'],
|
||||
['POST', '/api/providers/up/key'], ['POST', '/api/providers/up/probe'],
|
||||
['DELETE', '/api/cache'],
|
||||
];
|
||||
|
||||
for (const [method, path] of routes) {
|
||||
const res = await fetch(`${h.base}${path}`, { method, headers: authed(APP), ...(method === 'GET' ? {} : { body: '{}' }) });
|
||||
assert.equal(res.status, 403, `${method} ${path} should be forbidden to an app key`);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'admin_required');
|
||||
}
|
||||
});
|
||||
|
||||
it('still lets the app key do inference', async () => {
|
||||
const h = await harness(cfg);
|
||||
const res = await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: authed(APP),
|
||||
body: JSON.stringify({ model: 'fast', messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
assert.equal(res.status, 200);
|
||||
});
|
||||
|
||||
it('lets the admin key through', async () => {
|
||||
const h = await harness(cfg);
|
||||
assert.equal((await fetch(`${h.base}/api/stats`, { headers: authed(ADMIN) })).status, 200);
|
||||
});
|
||||
|
||||
it('refuses /api entirely on a public bind with no admin key', async () => {
|
||||
// Otherwise an exposed gateway with only inference keys hands them the box.
|
||||
const h = await harness({ keys: [APP], adminKey: null, host: '0.0.0.0' });
|
||||
const res = await fetch(`${h.base}/api/logs`, { headers: authed(APP) });
|
||||
assert.equal(res.status, 403);
|
||||
assert.match(((await res.json()) as { error: { message: string } }).error.message, /FREEWAY_ADMIN_KEY/);
|
||||
});
|
||||
|
||||
it('stays convenient on loopback with no admin key', async () => {
|
||||
const h = await harness({ keys: [APP], adminKey: null, host: '127.0.0.1' });
|
||||
assert.equal((await fetch(`${h.base}/api/stats`, { headers: authed(APP) })).status, 200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('C2 — CORS is an allowlist, never a wildcard', () => {
|
||||
it('sends no CORS headers by default', async () => {
|
||||
const h = await harness({ keys: [APP] });
|
||||
const res = await fetch(`${h.base}/v1/models`, { headers: { ...authed(APP), origin: 'https://evil.example' } });
|
||||
assert.equal(res.headers.get('access-control-allow-origin'), null);
|
||||
});
|
||||
|
||||
it('refuses an origin that is not on the list', async () => {
|
||||
const h = await harness({ keys: [APP], corsOrigins: ['https://app.example'] });
|
||||
const res = await fetch(`${h.base}/v1/models`, { headers: { ...authed(APP), origin: 'https://evil.example' } });
|
||||
assert.equal(res.headers.get('access-control-allow-origin'), null);
|
||||
});
|
||||
|
||||
it('echoes only a listed origin, and varies on it', async () => {
|
||||
const h = await harness({ keys: [APP], corsOrigins: ['https://app.example'] });
|
||||
const res = await fetch(`${h.base}/v1/models`, { headers: { ...authed(APP), origin: 'https://app.example' } });
|
||||
assert.equal(res.headers.get('access-control-allow-origin'), 'https://app.example');
|
||||
assert.match(res.headers.get('vary') ?? '', /Origin/);
|
||||
});
|
||||
|
||||
it('rejects "*" in configuration rather than honouring it', () => {
|
||||
const { config, issues } = parseConfig({ corsOrigins: ['*', 'https://ok.example'] }, {});
|
||||
assert.deepEqual(config.corsOrigins, ['https://ok.example']);
|
||||
assert.ok(issues.some((i) => i.path === 'corsOrigins'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('C3 — sessions belong to the key that made them', () => {
|
||||
it('refuses to read another key’s session', async () => {
|
||||
const h = await harness({ keys: [APP, 'fw-other-key'], adminKey: ADMIN });
|
||||
|
||||
await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: authed(APP),
|
||||
body: JSON.stringify({ model: 'fast', session: 'private', messages: [{ role: 'user', content: 'my bank pin is 1234' }] }),
|
||||
});
|
||||
|
||||
const res = await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: authed('fw-other-key'),
|
||||
body: JSON.stringify({ model: 'fast', session: 'private', messages: [{ role: 'user', content: 'what did I say?' }] }),
|
||||
});
|
||||
|
||||
assert.equal(res.status, 403);
|
||||
assert.equal(((await res.json()) as { error: { code: string } }).error.code, 'session_forbidden');
|
||||
});
|
||||
|
||||
it('lets the owner continue its own session', async () => {
|
||||
const h = await harness({ keys: [APP], adminKey: ADMIN });
|
||||
const body = (content: string) => JSON.stringify({ model: 'fast', session: 'mine', messages: [{ role: 'user', content }] });
|
||||
|
||||
assert.equal((await fetch(`${h.base}/v1/chat/completions`, { method: 'POST', headers: authed(APP), body: body('one') })).status, 200);
|
||||
assert.equal((await fetch(`${h.base}/v1/chat/completions`, { method: 'POST', headers: authed(APP), body: body('two') })).status, 200);
|
||||
assert.equal(h.context.messages('mine').length, 4);
|
||||
});
|
||||
|
||||
it('records the owner, masked', async () => {
|
||||
const h = await harness({ keys: [APP], adminKey: ADMIN });
|
||||
await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: authed(APP),
|
||||
body: JSON.stringify({ model: 'fast', session: 'owned', messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
const owner = h.context.info('owned')?.owner;
|
||||
assert.ok(owner, 'session should be claimed');
|
||||
assert.ok(!owner.includes('app-key'), 'the owner must be a mask, not the key');
|
||||
});
|
||||
});
|
||||
|
||||
describe('H2 — live log subscriptions are capped', () => {
|
||||
it('refuses beyond the cap instead of accumulating listeners', async () => {
|
||||
const h = await harness();
|
||||
const controllers: AbortController[] = [];
|
||||
let refused = 0;
|
||||
|
||||
for (let i = 0; i < 20; i++) {
|
||||
const controller = new AbortController();
|
||||
controllers.push(controller);
|
||||
const res = await fetch(`${h.base}/api/events`, { signal: controller.signal });
|
||||
if (res.status === 503) {
|
||||
refused += 1;
|
||||
break;
|
||||
}
|
||||
// Start reading so the subscription actually registers server-side.
|
||||
void res.body?.getReader().read();
|
||||
}
|
||||
for (const c of controllers) c.abort();
|
||||
assert.ok(refused > 0, 'an unbounded number of subscribers was accepted');
|
||||
});
|
||||
});
|
||||
|
||||
describe('H3 — response headers cannot be poisoned by input', () => {
|
||||
it('survives a session id full of illegal header bytes', async () => {
|
||||
// These used to reach writeHead raw and throw, turning a good response
|
||||
// into a remote 500.
|
||||
const h = await harness();
|
||||
for (const id of ['sess\r\nX-Injected: yes', 'sess→arrow', 'sess null']) {
|
||||
const res = await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model: 'fast', session: id, messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
assert.equal(res.status, 200, `session id ${JSON.stringify(id)} should not break the response`);
|
||||
assert.equal(res.headers.get('x-injected'), null, 'no header was injected');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('M1 — data at rest is not world-readable', () => {
|
||||
it('creates the sqlite files 0600', async () => {
|
||||
const h = await harness();
|
||||
await fetch(`${h.base}/v1/chat/completions`, {
|
||||
method: 'POST', headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ model: 'fast', session: 's', temperature: 0, messages: [{ role: 'user', content: 'hi' }] }),
|
||||
});
|
||||
|
||||
for (const file of ['freeway.db', 'cache.db']) {
|
||||
const mode = statSync(join(h.dataDir, file)).mode & 0o777;
|
||||
assert.equal(mode, 0o600, `${file} is 0${mode.toString(8)}, expected 0600`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('M2 — key comparison leaks neither content nor length', () => {
|
||||
it('rejects keys of every length identically', async () => {
|
||||
const h = await harness({ keys: ['fw-the-real-key-abcdefghijk'] });
|
||||
for (const wrong of ['x', 'fw-', 'fw-the-real-key-abcdefghij', 'fw-the-real-key-abcdefghijkX', 'x'.repeat(500)]) {
|
||||
const res = await fetch(`${h.base}/v1/models`, { headers: { authorization: `Bearer ${wrong}` } });
|
||||
assert.equal(res.status, 401, `"${wrong.slice(0, 12)}…" should be rejected`);
|
||||
}
|
||||
assert.equal((await fetch(`${h.base}/v1/models`, { headers: authed('fw-the-real-key-abcdefghijk') })).status, 200);
|
||||
});
|
||||
});
|
||||
|
||||
describe('still no way to reach a raw provider key', () => {
|
||||
it('keeps provider keys out of every admin response', async () => {
|
||||
const h = await harness({ keys: [APP], adminKey: ADMIN });
|
||||
for (const path of ['/api/providers', '/api/stats', '/api/logs', '/api/alerts']) {
|
||||
const text = await (await fetch(`${h.base}${path}`, { headers: authed(ADMIN) })).text();
|
||||
assert.ok(!text.includes('uk-000000000001'), `${path} leaked a provider key`);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,242 @@
|
||||
/**
|
||||
* End-to-end sessions: the client sends one message, the gateway rebuilds the
|
||||
* thread, and the thread survives a failover to a much smaller model.
|
||||
*/
|
||||
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import type { Server } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { ContextStore, Registry, Router, Store, Tokenizer, parseConfig, silentLogger, type FreewayConfig } from '@freeway/core';
|
||||
|
||||
import { createGateway } from '../src/server.ts';
|
||||
import { createMockUpstream } from '../../../scripts/mock-upstream.js';
|
||||
|
||||
const cleanups: (() => unknown)[] = [];
|
||||
after(async () => {
|
||||
for (const c of cleanups.reverse()) await c();
|
||||
});
|
||||
|
||||
interface Mock {
|
||||
url: string;
|
||||
script(steps: unknown[]): void;
|
||||
callsTo(path: string): { body: { messages?: { role: string; content: string }[]; model?: string } }[];
|
||||
close(): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface Harness {
|
||||
base: string;
|
||||
big: Mock;
|
||||
small: Mock;
|
||||
context: ContextStore;
|
||||
store: Store;
|
||||
}
|
||||
|
||||
/** `big` has a 128k window, `small` has 8k — the shape of the real scenario. */
|
||||
async function harness(over: Partial<FreewayConfig> = {}): Promise<Harness> {
|
||||
const big = (await createMockUpstream({ name: 'big' })) as Mock;
|
||||
const small = (await createMockUpstream({ name: 'small' })) as Mock;
|
||||
cleanups.push(() => big.close(), () => small.close());
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-sess-'));
|
||||
cleanups.push(() => rmSync(dir, { recursive: true, force: true }));
|
||||
|
||||
writeFileSync(join(dir, 'big.json'), JSON.stringify({
|
||||
id: 'big', priority: 10, baseUrl: big.url,
|
||||
auth: { type: 'bearer', envKeys: ['BIG_KEY'] }, limits: { rpm: 100 },
|
||||
models: [{ id: 'big-chat', alias: ['fast'], context: 128000, caps: ['chat'] }],
|
||||
}));
|
||||
writeFileSync(join(dir, 'small.json'), JSON.stringify({
|
||||
id: 'small', priority: 20, baseUrl: small.url,
|
||||
auth: { type: 'bearer', envKeys: ['SMALL_KEY'] }, limits: { rpm: 100 },
|
||||
models: [{ id: 'small-chat', alias: ['fast'], context: 8192, caps: ['chat'] }],
|
||||
}));
|
||||
|
||||
const registry = new Registry({ dir, env: { BIG_KEY: 'bk-000000000001', SMALL_KEY: 'sk-000000000001' }, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
Object.assign(config, over);
|
||||
const router = new Router(registry, store, config);
|
||||
const context = new ContextStore(':memory:');
|
||||
cleanups.push(() => context.close());
|
||||
|
||||
const server: Server = createGateway({
|
||||
registry, store, router, config, logger: silentLogger,
|
||||
contextStore: context, tokenizer: new Tokenizer(),
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const addr = server.address();
|
||||
const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
|
||||
cleanups.push(() => new Promise((r) => server.close(() => r(undefined))));
|
||||
|
||||
return { base: `http://127.0.0.1:${port}`, big, small, context, store };
|
||||
}
|
||||
|
||||
async function chat(base: string, body: unknown): Promise<Response> {
|
||||
return fetch(`${base}/v1/chat/completions`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
}
|
||||
|
||||
const say = (text: string) => ({ role: 'user', content: text });
|
||||
|
||||
describe('sessions — server-side conversation', () => {
|
||||
it('rebuilds the thread from just the new message', async () => {
|
||||
const h = await harness();
|
||||
|
||||
await chat(h.base, { model: 'fast', session: 'thread-1', messages: [say('my name is Ada')] });
|
||||
const res = await chat(h.base, { model: 'fast', session: 'thread-1', messages: [say('what is my name?')] });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-session'), 'thread-1');
|
||||
|
||||
// The upstream saw the whole conversation, not just the last line.
|
||||
const calls = h.big.callsTo('/chat/completions');
|
||||
const sent = calls[calls.length - 1]?.body.messages ?? [];
|
||||
assert.ok(sent.length >= 3, `expected rehydrated history, got ${sent.length} messages`);
|
||||
assert.ok(sent.some((m) => m.content.includes('my name is Ada')), 'the earlier turn was replayed');
|
||||
});
|
||||
|
||||
it('strips `session` before forwarding — no provider ever sees it', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, { model: 'fast', session: 'thread-2', messages: [say('hi')] });
|
||||
|
||||
const body = h.big.callsTo('/chat/completions')[0]?.body as Record<string, unknown>;
|
||||
assert.equal(body['session'], undefined);
|
||||
});
|
||||
|
||||
it('persists the assistant reply so the next turn sees it', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, { model: 'fast', session: 'thread-3', messages: [say('hello')] });
|
||||
|
||||
const stored = h.context.messages('thread-3');
|
||||
assert.deepEqual(stored.map((m) => m.role), ['user', 'assistant']);
|
||||
assert.match(stored[1]?.content ?? '', /\[big\]/);
|
||||
});
|
||||
|
||||
it('keeps sessions isolated', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, { model: 'fast', session: 'a', messages: [say('secret for A')] });
|
||||
await chat(h.base, { model: 'fast', session: 'b', messages: [say('secret for B')] });
|
||||
|
||||
const calls = h.big.callsTo('/chat/completions');
|
||||
const forB = calls[calls.length - 1]?.body.messages ?? [];
|
||||
assert.ok(!forB.some((m) => m.content.includes('secret for A')), 'session A must not leak into session B');
|
||||
});
|
||||
|
||||
it('behaves like a plain OpenAI endpoint with no session id', async () => {
|
||||
const h = await harness();
|
||||
const res = await chat(h.base, { model: 'fast', messages: [say('stateless please')] });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-session'), null);
|
||||
assert.equal(h.context.list().length, 0, 'nothing was stored');
|
||||
});
|
||||
});
|
||||
|
||||
describe('sessions — surviving a failover to a smaller window', () => {
|
||||
it('compacts the history to fit the model that actually serves it', async () => {
|
||||
const h = await harness({ context: { enabled: true, maxTurns: 4, reserveOutputTokens: 512, compactModel: 'fast', autoCompactAt: 0.8 } });
|
||||
|
||||
// Build a long thread on the 128k model.
|
||||
const long = 'This is a reasonably long turn about the project. '.repeat(120);
|
||||
for (let i = 0; i < 12; i++) {
|
||||
await chat(h.base, { model: 'fast', session: 'long', messages: [say(`turn ${i}: ${long}`)] });
|
||||
}
|
||||
|
||||
const beforeCalls = h.big.callsTo('/chat/completions').length;
|
||||
const sentToBig = h.big.callsTo('/chat/completions')[beforeCalls - 1]?.body.messages ?? [];
|
||||
assert.ok(sentToBig.length > 5, 'the big model received the full history');
|
||||
|
||||
// The big provider runs out; everything now has to go to the 8k model.
|
||||
h.big.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const res = await chat(h.base, { model: 'fast', session: 'long', messages: [say('given all that, what should I do?')] });
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'small', 'it failed over');
|
||||
|
||||
const note = res.headers.get('x-freeway-context');
|
||||
assert.ok(note, 'the refit is reported on the response');
|
||||
assert.match(note, /refit=(window|compact|hard-truncate|drop-tool-noise)/);
|
||||
assert.match(note, /tokens=\d+->\d+/);
|
||||
|
||||
// The reshaped history genuinely fits the smaller window.
|
||||
const smallCalls = h.small.callsTo('/chat/completions');
|
||||
const sentToSmall = smallCalls[smallCalls.length - 1]?.body.messages ?? [];
|
||||
const chars = sentToSmall.reduce((n, m) => n + String(m.content).length, 0);
|
||||
assert.ok(chars / 4 < 8192, `history must fit 8k, estimated ${Math.round(chars / 4)} tokens`);
|
||||
|
||||
// And the newest question survived intact — that is the request itself.
|
||||
assert.ok(sentToSmall.some((m) => m.content.includes('given all that, what should I do?')));
|
||||
});
|
||||
|
||||
it('reports a refit even without a session, so plain failover still fits', async () => {
|
||||
// A client that manages its own history still benefits: the array it sent
|
||||
// is reshaped for whichever model ends up serving the call.
|
||||
const h = await harness();
|
||||
h.big.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const turn = 'padding '.repeat(300); // ~600 tokens each
|
||||
const messages = [{ role: 'system', content: 'be helpful' }];
|
||||
for (let i = 0; i < 20; i++) {
|
||||
messages.push({ role: i % 2 === 0 ? 'user' : 'assistant', content: `${i}: ${turn}` });
|
||||
}
|
||||
messages.push(say('and finally, a short question'));
|
||||
|
||||
const res = await chat(h.base, { model: 'fast', messages });
|
||||
|
||||
assert.equal(res.status, 200);
|
||||
assert.equal(res.headers.get('x-freeway-provider'), 'small');
|
||||
const note = res.headers.get('x-freeway-context');
|
||||
assert.ok(note, 'stateless requests are refit too');
|
||||
assert.ok(!note.includes('refit=passthrough'), `it genuinely had to reshape, got "${note}"`);
|
||||
|
||||
const smallCalls = h.small.callsTo('/chat/completions');
|
||||
const sent = smallCalls[smallCalls.length - 1]?.body.messages ?? [];
|
||||
const chars = sent.reduce((n, m) => n + String(m.content).length, 0);
|
||||
assert.ok(chars / 4 < 8192, `must fit the 8k window, estimated ${Math.round(chars / 4)} tokens`);
|
||||
assert.ok(sent.some((m) => m.content.includes('and finally, a short question')), 'the actual request survived');
|
||||
});
|
||||
|
||||
it('returns 413 when the newest message alone cannot fit', async () => {
|
||||
const h = await harness();
|
||||
// Only the 8k model is available, and the message needs far more than that.
|
||||
h.big.script([{ status: 429, repeat: Infinity }]);
|
||||
|
||||
const huge = 'x'.repeat(200_000);
|
||||
const res = await chat(h.base, { model: 'fast', messages: [say(huge)] });
|
||||
|
||||
assert.equal(res.status, 413);
|
||||
const body = (await res.json()) as { error: { code: string }; freeway: { required: number; available: number } };
|
||||
assert.equal(body.error.code, 'context_too_large');
|
||||
assert.ok(body.freeway.required > body.freeway.available);
|
||||
});
|
||||
});
|
||||
|
||||
describe('sessions — API', () => {
|
||||
it('lists and inspects sessions, then deletes one', async () => {
|
||||
const h = await harness();
|
||||
await chat(h.base, { model: 'fast', session: 'inspect-me', messages: [say('hello there')] });
|
||||
|
||||
const list = (await (await fetch(`${h.base}/api/sessions`)).json()) as { sessions: { id: string; messages: number }[] };
|
||||
assert.ok(list.sessions.some((s) => s.id === 'inspect-me'));
|
||||
|
||||
const one = (await (await fetch(`${h.base}/api/sessions/inspect-me`)).json()) as { messages: { role: string }[] };
|
||||
assert.deepEqual(one.messages.map((m) => m.role), ['user', 'assistant']);
|
||||
|
||||
const del = await fetch(`${h.base}/api/sessions/inspect-me`, { method: 'DELETE' });
|
||||
assert.equal(del.status, 200);
|
||||
assert.equal(h.context.info('inspect-me'), null);
|
||||
});
|
||||
|
||||
it('404s an unknown session', async () => {
|
||||
const h = await harness();
|
||||
assert.equal((await fetch(`${h.base}/api/sessions/nope`)).status, 404);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user