Freeway: self-hosted OpenAI-compatible gateway for free-tier providers
One endpoint in front of every free LLM tier, with automatic failover and no provider key ever leaving the box. Providers are data, not code: adding one is a JSON file plus an env var, never a TypeScript change. Ships 12 providers pre-written; `verify-provider --apply` derives the live model list from your own key. - core: registry, bucketed quota meter (rolling + UTC-calendar windows), key-pool store, 6-step model router with diagnostic rejection reasons - gateway: node:http, failover that commits to a stream only after 200 headers, SSE passthrough with usage sniffing, virtual keys with admin/inference roles - context engine: sqlite sessions and a 5-rung refit ladder, so a 128k thread keeps working when it fails over to an 8k model - cache: exact (sha256, safe-by-default) + opt-in semantic (flat Float32Array) - guard: inbound limits, quota forecasting, limit calibration from upstream headers, key health, secret/PII/injection scanning - dashboard: one self-contained HTML file, live SSE request log, token usage - sdk: zero-dep typed client, drop-in for the OpenAI SDK Zero runtime dependencies. 326 tests, 27-check smoke suite against a scripted mock upstream so routing is testable without spending quota. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Aryan SinghandClaude Opus 5 committed
commit
cb985dcf9c
86 files changed
+16350
No files matched your search
@@ -0,0 +1,366 @@
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { parseConfig } from '../src/config.ts';
|
||||
import { Registry } from '../src/registry.ts';
|
||||
import { Router } from '../src/router.ts';
|
||||
import { Store } from '../src/store.ts';
|
||||
import type { FreewayConfig, Strategy } from '../src/types.ts';
|
||||
import { silentLogger } from '../src/util.ts';
|
||||
|
||||
const dirs: string[] = [];
|
||||
after(() => {
|
||||
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/**
|
||||
* Two providers with overlapping aliases, a two-key pool, a Cloudflare-shaped
|
||||
* slashy model id, and models with differing caps and context windows.
|
||||
*/
|
||||
const FIXTURES = {
|
||||
'alpha.json': {
|
||||
id: 'alpha',
|
||||
priority: 10,
|
||||
baseUrl: 'https://alpha.test/v1',
|
||||
auth: { type: 'bearer', envKeys: ['ALPHA_KEY'] },
|
||||
limits: { rpm: 10 },
|
||||
models: [
|
||||
{ id: 'alpha-chat', alias: ['fast'], context: 128000, caps: ['chat', 'tools', 'json'], priority: 10 },
|
||||
{ id: 'alpha-tiny', alias: [], context: 4096, caps: ['chat'], priority: 20 },
|
||||
{ id: 'alpha-embed', alias: ['embed'], context: 8000, caps: ['embed'], priority: 10 },
|
||||
],
|
||||
},
|
||||
'beta.json': {
|
||||
id: 'beta',
|
||||
priority: 20,
|
||||
baseUrl: 'https://beta.test/v1',
|
||||
auth: { type: 'bearer', envKeys: ['BETA_KEY'] },
|
||||
limits: { rpm: 10 },
|
||||
models: [
|
||||
{ id: '@cf/meta/llama-3.1-8b-instruct', alias: ['fast'], context: 8192, caps: ['chat'], priority: 10 },
|
||||
{ id: 'beta-vision', alias: [], context: 32000, caps: ['chat', 'vision'], priority: 20 },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
interface Harness {
|
||||
registry: Registry;
|
||||
store: Store;
|
||||
router: Router;
|
||||
config: FreewayConfig;
|
||||
}
|
||||
|
||||
function harness(overrides: Partial<FreewayConfig> = {}, env?: Record<string, string | undefined>): Harness {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-router-'));
|
||||
dirs.push(dir);
|
||||
for (const [name, body] of Object.entries(FIXTURES)) writeFileSync(join(dir, name), JSON.stringify(body));
|
||||
|
||||
const registry = new Registry({
|
||||
dir,
|
||||
env: env ?? { ALPHA_KEY: 'ak-000000000001,ak-000000000002', BETA_KEY: 'bk-000000000001' },
|
||||
logger: silentLogger,
|
||||
});
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
Object.assign(config, overrides);
|
||||
return { registry, store, router: new Router(registry, store, config), config };
|
||||
}
|
||||
|
||||
function refs(result: { candidates: { model: { ref: string } }[] }): string[] {
|
||||
return [...new Set(result.candidates.map((c) => c.model.ref))];
|
||||
}
|
||||
|
||||
describe('Router — model string resolution', () => {
|
||||
it('1. auto returns every chat model and excludes embeddings', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'auto' });
|
||||
assert.equal(r.resolution, 'auto');
|
||||
assert.equal(r.matched, true);
|
||||
assert.ok(refs(r).includes('alpha/alpha-chat'));
|
||||
assert.ok(!refs(r).includes('alpha/alpha-embed'), 'embedding models never serve chat');
|
||||
});
|
||||
|
||||
it('1. treats an empty model string as auto', () => {
|
||||
const { router } = harness();
|
||||
assert.equal(router.route({ model: '' }).resolution, 'auto');
|
||||
assert.equal(router.route({ model: 'freeway' }).resolution, 'auto');
|
||||
});
|
||||
|
||||
it('2. pins provider/model, splitting on the FIRST slash only', () => {
|
||||
// A Cloudflare id has two more slashes of its own; naive splitting breaks it.
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'beta/@cf/meta/llama-3.1-8b-instruct' });
|
||||
assert.equal(r.matched, true);
|
||||
assert.equal(r.resolution, 'pin:beta/@cf/meta/llama-3.1-8b-instruct');
|
||||
assert.deepEqual(refs(r), ['beta/@cf/meta/llama-3.1-8b-instruct']);
|
||||
});
|
||||
|
||||
it('2. pins a whole provider with provider/*', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'alpha/*' });
|
||||
assert.equal(r.resolution, 'pin:alpha/*');
|
||||
assert.deepEqual(refs(r).sort(), ['alpha/alpha-chat', 'alpha/alpha-tiny']);
|
||||
});
|
||||
|
||||
it('2. an unknown model under a known provider does not match', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'alpha/no-such-model' });
|
||||
assert.equal(r.matched, false);
|
||||
});
|
||||
|
||||
it('3. config aliases beat registry aliases', () => {
|
||||
const { router } = harness({ aliases: { fast: 'beta/beta-vision' } });
|
||||
const r = router.route({ model: 'fast' });
|
||||
assert.equal(r.resolution, 'alias:fast->beta/beta-vision');
|
||||
assert.deepEqual(refs(r), ['beta/beta-vision']);
|
||||
});
|
||||
|
||||
it('keeps every resolution string safe to put in an HTTP header', () => {
|
||||
// `resolution` is served as x-freeway-route. A non-latin-1 character here
|
||||
// makes writeHead throw and turns a good response into a 500 — which is
|
||||
// exactly what a Unicode arrow in this string used to do.
|
||||
const { router } = harness({ aliases: { fast: 'beta/beta-vision', missing: 'nope/nope' } });
|
||||
for (const model of ['auto', 'fast', 'missing', 'alpha/*', 'alpha-tiny', 'llama-3.1-8b', 'no-such-model']) {
|
||||
const { resolution } = router.route({ model });
|
||||
assert.ok(/^[\x20-\x7E]*$/.test(resolution), `resolution "${resolution}" for "${model}" is not header-safe`);
|
||||
}
|
||||
});
|
||||
|
||||
it('4. a registry alias collects every model that declares it', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'fast' });
|
||||
assert.equal(r.resolution, 'alias:fast');
|
||||
assert.deepEqual(refs(r).sort(), ['alpha/alpha-chat', 'beta/@cf/meta/llama-3.1-8b-instruct']);
|
||||
});
|
||||
|
||||
it('5. matches an exact upstream model id across providers', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'alpha-tiny' });
|
||||
assert.equal(r.resolution, 'model:alpha-tiny');
|
||||
assert.deepEqual(refs(r), ['alpha/alpha-tiny']);
|
||||
});
|
||||
|
||||
it('6. fuzzy substring finds a slashy id from its bare name', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'llama-3.1-8b' });
|
||||
assert.equal(r.resolution, 'fuzzy:llama-3.1-8b');
|
||||
assert.deepEqual(refs(r), ['beta/@cf/meta/llama-3.1-8b-instruct']);
|
||||
});
|
||||
|
||||
it('reports no match with suggestions rather than guessing', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'alpha-chatt' });
|
||||
// Close enough to be a typo, so it fuzzily matches nothing but suggests.
|
||||
if (!r.matched) {
|
||||
assert.ok(r.suggestions.includes('alpha/alpha-chat'), `expected a suggestion, got ${r.suggestions.join(',')}`);
|
||||
}
|
||||
});
|
||||
|
||||
it('suggests the closest ids for a wholly unknown model', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'gpt-4o' });
|
||||
assert.equal(r.matched, false);
|
||||
assert.equal(r.candidates.length, 0);
|
||||
});
|
||||
|
||||
it('routes embeddings to embedding models only', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'auto', embedding: true });
|
||||
assert.deepEqual(refs(r), ['alpha/alpha-embed']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Router — capability and context filtering', () => {
|
||||
it('blocks a model that lacks a required capability, and says which', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'fast', requiredCaps: ['tools'] });
|
||||
assert.deepEqual(refs(r), ['alpha/alpha-chat']);
|
||||
|
||||
const blocked = r.blocked.find((b) => b.modelId === '@cf/meta/llama-3.1-8b-instruct');
|
||||
assert.ok(blocked);
|
||||
assert.equal(blocked.reason, 'missing capability: tools');
|
||||
});
|
||||
|
||||
it('blocks a model whose window is too small, with both numbers', () => {
|
||||
const { router } = harness();
|
||||
const r = router.route({ model: 'alpha/*', minContext: 32000 });
|
||||
assert.deepEqual(refs(r), ['alpha/alpha-chat']);
|
||||
|
||||
const blocked = r.blocked.find((b) => b.modelId === 'alpha-tiny');
|
||||
assert.ok(blocked);
|
||||
assert.equal(blocked.reason, 'context window 4096 < required 32000');
|
||||
});
|
||||
|
||||
it('refuses to promise a context window it does not know', () => {
|
||||
const { router, registry } = harness();
|
||||
const m = registry.get('alpha')?.models.find((x) => x.spec.id === 'alpha-tiny');
|
||||
assert.ok(m);
|
||||
m.spec.context = null;
|
||||
|
||||
const r = router.route({ model: 'alpha/alpha-tiny', minContext: 1000 });
|
||||
assert.equal(r.candidates.length, 0);
|
||||
assert.match(r.blocked[0]?.reason ?? '', /context window unknown/);
|
||||
});
|
||||
|
||||
it('blocks an unconfigured provider with the env var to set', () => {
|
||||
const { router } = harness({}, { ALPHA_KEY: 'ak-000000000001' });
|
||||
const r = router.route({ model: 'fast' });
|
||||
const blocked = r.blocked.find((b) => b.providerId === 'beta');
|
||||
assert.ok(blocked);
|
||||
assert.equal(blocked.reason, 'provider not configured: no API key found (set one of: BETA_KEY)');
|
||||
});
|
||||
|
||||
it('blocks a disabled provider', () => {
|
||||
const { router, registry } = harness();
|
||||
registry.setEnabled('beta', false);
|
||||
const r = router.route({ model: 'fast' });
|
||||
assert.deepEqual(refs(r), ['alpha/alpha-chat']);
|
||||
assert.ok(r.blocked.some((b) => b.providerId === 'beta' && b.reason === 'provider disabled'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('Router — quota and cooldown', () => {
|
||||
it('moves a quota-exhausted provider into blocked with a diagnostic reason', () => {
|
||||
const { router, store } = harness();
|
||||
for (let i = 0; i < 10; i++) store.recordSuccess('alpha', 'alpha#0', 10);
|
||||
|
||||
const r = router.route({ model: 'fast' });
|
||||
assert.deepEqual(refs(r), ['beta/@cf/meta/llama-3.1-8b-instruct'], 'failed over to the other provider');
|
||||
|
||||
const blocked = r.blocked.find((b) => b.providerId === 'alpha');
|
||||
assert.ok(blocked);
|
||||
assert.match(blocked.reason, /rpm limit reached \(10\/10 last 60s\)/);
|
||||
assert.ok((blocked.retryAfterMs ?? 0) > 0, 'tells the caller when to come back');
|
||||
});
|
||||
|
||||
it('a 429 benches one key but the pool stays viable', () => {
|
||||
const { router, store } = harness();
|
||||
store.recordFailure('alpha', 'alpha#0', { status: 429, message: 'rate limited' });
|
||||
|
||||
const r = router.route({ model: 'alpha/alpha-chat' });
|
||||
const keys = r.candidates.map((c) => c.key.id);
|
||||
assert.ok(keys.includes('alpha#1'), 'the healthy key is still offered');
|
||||
assert.ok(!keys.includes('alpha#0'), 'the benched key is not');
|
||||
assert.equal(r.blocked.length, 0, 'the provider is not blocked at all');
|
||||
});
|
||||
|
||||
it('reports every key when the whole pool is down, with masked ids', () => {
|
||||
const { router, store } = harness();
|
||||
store.recordFailure('alpha', 'alpha#0', { status: 429, message: 'rate limited' });
|
||||
store.recordFailure('alpha', 'alpha#1', { status: 401, message: 'unauthorized' });
|
||||
|
||||
const r = router.route({ model: 'alpha/alpha-chat' });
|
||||
assert.equal(r.candidates.length, 0);
|
||||
const blocked = r.blocked[0];
|
||||
assert.ok(blocked);
|
||||
assert.match(blocked.reason, /all 2 keys unavailable/);
|
||||
assert.match(blocked.reason, /ak-0…0001/, 'keys appear masked, never raw');
|
||||
assert.ok(!blocked.reason.includes('ak-000000000001'), 'the raw key must never leak into an error');
|
||||
});
|
||||
|
||||
it('a 503 benches the provider and routing moves on', () => {
|
||||
const { router, store } = harness();
|
||||
store.recordFailure('alpha', 'alpha#0', { status: 503, message: '503 from upstream' });
|
||||
|
||||
const r = router.route({ model: 'fast' });
|
||||
assert.deepEqual(refs(r), ['beta/@cf/meta/llama-3.1-8b-instruct']);
|
||||
|
||||
const blocked = r.blocked.find((b) => b.providerId === 'alpha');
|
||||
assert.ok(blocked);
|
||||
assert.match(blocked.reason, /provider cooling down for \d+s: 503 from upstream/);
|
||||
});
|
||||
|
||||
it('leaves the whole blocked list as the error message when nothing is viable', () => {
|
||||
const { router, store } = harness();
|
||||
store.recordFailure('alpha', 'alpha#0', { status: 503, message: '503 from upstream' });
|
||||
store.recordFailure('beta', 'beta#0', { status: 503, message: 'connection reset' });
|
||||
|
||||
const r = router.route({ model: 'fast' });
|
||||
assert.equal(r.matched, true, 'the model string was understood');
|
||||
assert.equal(r.candidates.length, 0, 'but nothing can serve it');
|
||||
assert.equal(r.blocked.length, 2);
|
||||
for (const b of r.blocked) assert.ok(b.reason.length > 20, `reason must be diagnostic, got "${b.reason}"`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Router — strategies', () => {
|
||||
function order(strategy: Strategy, prep?: (h: Harness) => void): string[] {
|
||||
const h = harness({ strategy });
|
||||
prep?.(h);
|
||||
return refs(h.router.route({ model: 'fast' }));
|
||||
}
|
||||
|
||||
it('priority follows provider then model priority', () => {
|
||||
assert.deepEqual(order('priority'), ['alpha/alpha-chat', 'beta/@cf/meta/llama-3.1-8b-instruct']);
|
||||
});
|
||||
|
||||
it('least-used prefers the provider with the emptier window', () => {
|
||||
const result = order('least-used', ({ store }) => {
|
||||
for (let i = 0; i < 5; i++) store.recordSuccess('alpha', 'alpha#0', 10);
|
||||
});
|
||||
assert.deepEqual(result, ['beta/@cf/meta/llama-3.1-8b-instruct', 'alpha/alpha-chat']);
|
||||
});
|
||||
|
||||
it('fastest prefers the lower EWMA latency', () => {
|
||||
const result = order('fastest', ({ store }) => {
|
||||
store.recordSuccess('alpha', 'alpha#0', 900);
|
||||
store.recordSuccess('beta', 'beta#0', 40);
|
||||
});
|
||||
assert.deepEqual(result, ['beta/@cf/meta/llama-3.1-8b-instruct', 'alpha/alpha-chat']);
|
||||
});
|
||||
|
||||
it('fastest samples an untried provider rather than starving it', () => {
|
||||
// Optimistic initialisation: never-tried providers must get a chance to earn
|
||||
// a latency number, or the first provider measured wins forever.
|
||||
const result = order('fastest', ({ store }) => {
|
||||
store.recordSuccess('alpha', 'alpha#0', 900);
|
||||
});
|
||||
assert.equal(result[0], 'beta/@cf/meta/llama-3.1-8b-instruct');
|
||||
});
|
||||
|
||||
it('round-robin rotates the head of the list between calls', () => {
|
||||
const h = harness({ strategy: 'round-robin' });
|
||||
const first = refs(h.router.route({ model: 'fast' }))[0];
|
||||
const second = refs(h.router.route({ model: 'fast' }))[0];
|
||||
assert.notEqual(first, second, 'consecutive calls should not hit the same provider');
|
||||
});
|
||||
|
||||
it('rotates keys within a pool', () => {
|
||||
const h = harness();
|
||||
const first = h.router.route({ model: 'alpha/alpha-chat' }).candidates[0]?.key.id;
|
||||
const second = h.router.route({ model: 'alpha/alpha-chat' }).candidates[0]?.key.id;
|
||||
assert.notEqual(first, second, 'a pool should spread load, not hammer key #0');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Router — real shipped catalog', () => {
|
||||
const dir = join(import.meta.dirname, '..', '..', '..', 'providers');
|
||||
|
||||
function real(env: Record<string, string | undefined>): Router {
|
||||
const registry = new Registry({ dir, env, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
return new Router(registry, store, config);
|
||||
}
|
||||
|
||||
it('pins a real Cloudflare model id without mangling the slashes', () => {
|
||||
const router = real({ CLOUDFLARE_ACCOUNT_ID: 'a1', CLOUDFLARE_API_TOKEN: 'cf-abcdefghijkl' });
|
||||
const r = router.route({ model: 'cloudflare/@cf/meta/llama-3.1-8b-instruct' });
|
||||
assert.equal(r.matched, true);
|
||||
assert.equal(r.candidates.length, 1);
|
||||
assert.equal(r.candidates[0]?.model.spec.id, '@cf/meta/llama-3.1-8b-instruct');
|
||||
});
|
||||
|
||||
it('tells you exactly what to set when nothing is configured', () => {
|
||||
const router = real({});
|
||||
const r = router.route({ model: 'auto' });
|
||||
assert.equal(r.matched, true);
|
||||
assert.equal(r.candidates.length, 0);
|
||||
const reasons = r.blocked.map((b) => b.reason);
|
||||
assert.ok(reasons.some((x) => /MISTRAL_API_KEY/.test(x)));
|
||||
assert.ok(reasons.some((x) => /CLOUDFLARE_ACCOUNT_ID/.test(x)));
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user