Freeway: self-hosted OpenAI-compatible gateway for free-tier providers
One endpoint in front of every free LLM tier, with automatic failover and no provider key ever leaving the box. Providers are data, not code: adding one is a JSON file plus an env var, never a TypeScript change. Ships 12 providers pre-written; `verify-provider --apply` derives the live model list from your own key. - core: registry, bucketed quota meter (rolling + UTC-calendar windows), key-pool store, 6-step model router with diagnostic rejection reasons - gateway: node:http, failover that commits to a stream only after 200 headers, SSE passthrough with usage sniffing, virtual keys with admin/inference roles - context engine: sqlite sessions and a 5-rung refit ladder, so a 128k thread keeps working when it fails over to an 8k model - cache: exact (sha256, safe-by-default) + opt-in semantic (flat Float32Array) - guard: inbound limits, quota forecasting, limit calibration from upstream headers, key health, secret/PII/injection scanning - dashboard: one self-contained HTML file, live SSE request log, token usage - sdk: zero-dep typed client, drop-in for the OpenAI SDK Zero runtime dependencies. 326 tests, 27-check smoke suite against a scripted mock upstream so routing is testable without spending quota. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Aryan SinghandClaude Opus 5 committed
commit
cb985dcf9c
86 files changed
+16350
No files matched your search
@@ -0,0 +1,336 @@
|
||||
import { strict as assert } from 'node:assert';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { after, describe, it } from 'node:test';
|
||||
|
||||
import { Registry } from '../src/registry.ts';
|
||||
import { silentLogger } from '../src/util.ts';
|
||||
|
||||
const dirs: string[] = [];
|
||||
after(() => {
|
||||
for (const d of dirs) rmSync(d, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
/** Write a throwaway providers/ directory. Values may be raw strings for bad JSON. */
|
||||
function providerDir(files: Record<string, unknown>): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-reg-'));
|
||||
dirs.push(dir);
|
||||
for (const [name, body] of Object.entries(files)) {
|
||||
writeFileSync(join(dir, name), typeof body === 'string' ? body : JSON.stringify(body, null, 2));
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
const MINIMAL = {
|
||||
id: 'demo',
|
||||
baseUrl: 'https://api.demo.test/v1',
|
||||
auth: { type: 'bearer', envKeys: ['DEMO_KEY'] },
|
||||
models: [{ id: 'demo-small', alias: ['fast'], context: 8000, caps: ['chat'] }],
|
||||
};
|
||||
|
||||
function make(dir: string, env: Record<string, string | undefined>): Registry {
|
||||
return new Registry({ dir, env, logger: silentLogger });
|
||||
}
|
||||
|
||||
describe('Registry — loading', () => {
|
||||
it('loads a valid provider file', () => {
|
||||
const reg = make(providerDir({ 'demo.json': MINIMAL }), { DEMO_KEY: 'sk-abcdefghijklmnop' });
|
||||
const p = reg.get('demo');
|
||||
assert.ok(p);
|
||||
assert.equal(p.configured, true);
|
||||
assert.equal(p.configError, null);
|
||||
assert.equal(p.models.length, 1);
|
||||
assert.equal(p.models[0]?.ref, 'demo/demo-small');
|
||||
});
|
||||
|
||||
it('skips files starting with an underscore', () => {
|
||||
// The JSON Schema lives in providers/ alongside the real files.
|
||||
const reg = make(
|
||||
providerDir({ 'demo.json': MINIMAL, '_schema.json': { $id: 'schema', type: 'object' } }),
|
||||
{ DEMO_KEY: 'sk-abcdefghijklmnop' },
|
||||
);
|
||||
assert.deepEqual(reg.all().map((p) => p.id), ['demo']);
|
||||
assert.equal(reg.issues().length, 0, 'skipped file must not produce an issue');
|
||||
});
|
||||
|
||||
it('skips a malformed JSON file without throwing, and keeps its neighbours', () => {
|
||||
const dir = providerDir({ 'demo.json': MINIMAL, 'broken.json': '{ "id": "broken", oops }' });
|
||||
let reg: Registry | undefined;
|
||||
assert.doesNotThrow(() => {
|
||||
reg = make(dir, { DEMO_KEY: 'sk-abcdefghijklmnop' });
|
||||
});
|
||||
assert.ok(reg);
|
||||
assert.deepEqual(reg.all().map((p) => p.id), ['demo'], 'valid provider still loads');
|
||||
|
||||
const issue = reg.issues().find((i) => i.file === 'broken.json');
|
||||
assert.ok(issue);
|
||||
assert.equal(issue.level, 'error');
|
||||
assert.match(issue.message, /invalid JSON/);
|
||||
});
|
||||
|
||||
it('reports the exact field path when a spec violates the schema', () => {
|
||||
const reg = make(
|
||||
providerDir({ 'bad.json': { id: 'bad', baseUrl: 'https://x.test', auth: { type: 'bearer', envKeys: ['K'] }, limits: { rpm: 'lots' } } }),
|
||||
{ K: 'k' },
|
||||
);
|
||||
assert.equal(reg.get('bad'), undefined, 'invalid provider is not loaded');
|
||||
const issue = reg.issues()[0];
|
||||
assert.ok(issue);
|
||||
assert.match(issue.message, /limits\.rpm/);
|
||||
});
|
||||
|
||||
it('warns about a misspelled field instead of silently ignoring it', () => {
|
||||
// `baseURL` vs `baseUrl` is the single worst failure mode of a JSON format.
|
||||
const reg = make(
|
||||
providerDir({ 'typo.json': { ...MINIMAL, id: 'typo', baseURL: 'https://oops.test' } }),
|
||||
{ DEMO_KEY: 'k1234567890abc' },
|
||||
);
|
||||
const warning = reg.issues().find((i) => i.level === 'warning');
|
||||
assert.ok(warning);
|
||||
assert.match(warning.message, /baseURL/);
|
||||
assert.match(warning.message, /did you mean "baseUrl"/);
|
||||
});
|
||||
|
||||
it('rejects two files claiming the same provider id', () => {
|
||||
const reg = make(
|
||||
providerDir({ 'a.json': MINIMAL, 'b.json': { ...MINIMAL, baseUrl: 'https://other.test/v1' } }),
|
||||
{ DEMO_KEY: 'k1234567890abc' },
|
||||
);
|
||||
assert.equal(reg.all().length, 1);
|
||||
assert.ok(reg.issues().some((i) => /duplicate provider id/.test(i.message)));
|
||||
});
|
||||
|
||||
it('survives a providers directory that does not exist', () => {
|
||||
assert.doesNotThrow(() => {
|
||||
const reg = new Registry({ dir: join(tmpdir(), 'freeway-does-not-exist-xyz'), env: {}, logger: silentLogger });
|
||||
assert.equal(reg.all().length, 0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Registry — environment interpolation', () => {
|
||||
it('resolves ${ENV_VAR} inside baseUrl', () => {
|
||||
// This is the Cloudflare case: the account id lives in the URL, and it must
|
||||
// stay pure data rather than becoming a special case in TypeScript.
|
||||
const reg = make(
|
||||
providerDir({
|
||||
'cf.json': {
|
||||
id: 'cf',
|
||||
baseUrl: 'https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT}/ai/v1',
|
||||
auth: { type: 'bearer', envKeys: ['CF_TOKEN'] },
|
||||
models: [{ id: '@cf/meta/llama-3.1-8b-instruct' }],
|
||||
},
|
||||
}),
|
||||
{ CF_ACCOUNT: 'acct123', CF_TOKEN: 'tok-abcdefghijkl' },
|
||||
);
|
||||
const p = reg.get('cf');
|
||||
assert.ok(p);
|
||||
assert.equal(p.baseUrl, 'https://api.cloudflare.com/client/v4/accounts/acct123/ai/v1');
|
||||
assert.equal(p.configured, true);
|
||||
});
|
||||
|
||||
it('resolves ${ENV_VAR} inside headers', () => {
|
||||
const reg = make(
|
||||
providerDir({ 'h.json': { ...MINIMAL, id: 'h', headers: { 'x-account': '${ACCT}', 'x-static': 'always' } } }),
|
||||
{ DEMO_KEY: 'k1234567890abc', ACCT: 'a-42' },
|
||||
);
|
||||
const p = reg.get('h');
|
||||
assert.ok(p);
|
||||
assert.deepEqual(p.headers, { 'x-account': 'a-42', 'x-static': 'always' });
|
||||
});
|
||||
|
||||
it('names the missing variable rather than failing silently', () => {
|
||||
const reg = make(
|
||||
providerDir({
|
||||
'cf.json': {
|
||||
id: 'cf',
|
||||
baseUrl: 'https://api.cloudflare.com/client/v4/accounts/${CF_ACCOUNT}/ai/v1',
|
||||
auth: { type: 'bearer', envKeys: ['CF_TOKEN'] },
|
||||
models: [{ id: 'm' }],
|
||||
},
|
||||
}),
|
||||
{ CF_TOKEN: 'tok-abcdefghijkl' },
|
||||
);
|
||||
const p = reg.get('cf');
|
||||
assert.ok(p);
|
||||
assert.equal(p.configured, false);
|
||||
assert.equal(p.configError, 'missing environment variable: CF_ACCOUNT');
|
||||
});
|
||||
|
||||
it('lists every missing variable at once', () => {
|
||||
const reg = make(
|
||||
providerDir({
|
||||
'multi.json': {
|
||||
id: 'multi',
|
||||
baseUrl: 'https://${REGION}.api.test/${VERSION}',
|
||||
auth: { type: 'bearer', envKeys: ['K'] },
|
||||
models: [{ id: 'm' }],
|
||||
},
|
||||
}),
|
||||
{ K: 'k1234567890abc' },
|
||||
);
|
||||
assert.equal(reg.get('multi')?.configError, 'missing environment variables: REGION, VERSION');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Registry — key pools', () => {
|
||||
it('splits one comma-separated env var into a pool', () => {
|
||||
// This has to work from a plain `docker run -e`, so commas are the grammar.
|
||||
const reg = make(providerDir({ 'demo.json': MINIMAL }), {
|
||||
DEMO_KEY: 'sk-aaaaaaaaaaaa,sk-bbbbbbbbbbbb , sk-cccccccccccc',
|
||||
});
|
||||
const p = reg.get('demo');
|
||||
assert.ok(p);
|
||||
assert.equal(p.keys.length, 3);
|
||||
assert.deepEqual(p.keys.map((k) => k.value), ['sk-aaaaaaaaaaaa', 'sk-bbbbbbbbbbbb', 'sk-cccccccccccc']);
|
||||
assert.deepEqual(p.keys.map((k) => k.id), ['demo#0', 'demo#1', 'demo#2']);
|
||||
});
|
||||
|
||||
it('pools keys across several env vars in declaration order', () => {
|
||||
const reg = make(
|
||||
providerDir({ 'demo.json': { ...MINIMAL, auth: { type: 'bearer', envKeys: ['DEMO_KEY', 'DEMO_KEY_2'] } } }),
|
||||
{ DEMO_KEY: 'sk-first-000000', DEMO_KEY_2: 'sk-second-00000,sk-third-000000' },
|
||||
);
|
||||
const p = reg.get('demo');
|
||||
assert.ok(p);
|
||||
assert.equal(p.keys.length, 3);
|
||||
assert.deepEqual(p.keys.map((k) => k.source), ['DEMO_KEY', 'DEMO_KEY_2', 'DEMO_KEY_2']);
|
||||
});
|
||||
|
||||
it('ignores blanks and duplicates', () => {
|
||||
const reg = make(providerDir({ 'demo.json': MINIMAL }), {
|
||||
DEMO_KEY: 'sk-aaaaaaaaaaaa,, ,sk-aaaaaaaaaaaa,sk-bbbbbbbbbbbb',
|
||||
});
|
||||
assert.equal(reg.get('demo')?.keys.length, 2);
|
||||
});
|
||||
|
||||
it('masks every key and never exposes the raw value in the mask', () => {
|
||||
const reg = make(providerDir({ 'demo.json': MINIMAL }), { DEMO_KEY: 'sk-supersecret-value-1234' });
|
||||
const key = reg.get('demo')?.keys[0];
|
||||
assert.ok(key);
|
||||
assert.equal(key.masked, 'sk-s…1234');
|
||||
assert.ok(!key.masked.includes('supersecret'));
|
||||
});
|
||||
|
||||
it('says which env var to set when no key is found', () => {
|
||||
const reg = make(
|
||||
providerDir({ 'demo.json': { ...MINIMAL, auth: { type: 'bearer', envKeys: ['DEMO_KEY', 'DEMO_KEY_2'] } } }),
|
||||
{},
|
||||
);
|
||||
const p = reg.get('demo');
|
||||
assert.ok(p);
|
||||
assert.equal(p.configured, false);
|
||||
assert.equal(p.configError, 'no API key found (set one of: DEMO_KEY, DEMO_KEY_2)');
|
||||
});
|
||||
|
||||
it('treats a keyless provider as configured', () => {
|
||||
const reg = make(
|
||||
providerDir({ 'local.json': { id: 'local', baseUrl: 'http://localhost:11434/v1', auth: { type: 'none' }, models: [{ id: 'llama3' }] } }),
|
||||
{},
|
||||
);
|
||||
const p = reg.get('local');
|
||||
assert.ok(p);
|
||||
assert.equal(p.configured, true);
|
||||
assert.equal(p.keys.length, 0);
|
||||
});
|
||||
|
||||
it('accepts runtime keys but lets env vars win', () => {
|
||||
const dir = providerDir({ 'demo.json': MINIMAL });
|
||||
const reg = new Registry({
|
||||
dir,
|
||||
env: { DEMO_KEY: 'sk-from-env-0000' },
|
||||
logger: silentLogger,
|
||||
extraKeys: { demo: 'sk-from-paste-00' },
|
||||
});
|
||||
const keys = reg.get('demo')?.keys ?? [];
|
||||
assert.equal(keys.length, 2);
|
||||
assert.equal(keys[0]?.source, 'DEMO_KEY', 'env key is tried first');
|
||||
assert.equal(keys[1]?.source, 'runtime');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Registry — selection surface', () => {
|
||||
const CATALOG = {
|
||||
'ready.json': {
|
||||
id: 'ready',
|
||||
priority: 10,
|
||||
baseUrl: 'https://ready.test/v1',
|
||||
auth: { type: 'bearer', envKeys: ['READY_KEY'] },
|
||||
models: [
|
||||
{ id: 'ready-fast', alias: ['fast'], caps: ['chat'] },
|
||||
{ id: 'ready-off', alias: ['fast'], caps: ['chat'], enabled: false },
|
||||
],
|
||||
},
|
||||
'nokey.json': {
|
||||
id: 'nokey',
|
||||
priority: 20,
|
||||
baseUrl: 'https://nokey.test/v1',
|
||||
auth: { type: 'bearer', envKeys: ['NOKEY_KEY'] },
|
||||
models: [{ id: 'nokey-model', alias: ['fast'] }],
|
||||
},
|
||||
};
|
||||
|
||||
it('usable() excludes providers with no key', () => {
|
||||
const reg = make(providerDir(CATALOG), { READY_KEY: 'k1234567890abc' });
|
||||
assert.deepEqual(reg.usable().map((p) => p.id), ['ready']);
|
||||
assert.equal(reg.all().length, 2, 'unconfigured provider still appears for the dashboard');
|
||||
});
|
||||
|
||||
it('usable() excludes a disabled provider, and the toggle survives reload', () => {
|
||||
const reg = make(providerDir(CATALOG), { READY_KEY: 'k1234567890abc' });
|
||||
assert.equal(reg.setEnabled('ready', false), true);
|
||||
assert.deepEqual(reg.usable().map((p) => p.id), []);
|
||||
|
||||
reg.reload();
|
||||
assert.equal(reg.get('ready')?.enabled, false, 'a dashboard toggle is not undone by re-reading disk');
|
||||
});
|
||||
|
||||
it('collects aliases from enabled models only', () => {
|
||||
const reg = make(providerDir(CATALOG), { READY_KEY: 'k1234567890abc', NOKEY_KEY: 'k1234567890abc' });
|
||||
const fast = reg.aliases().get('fast');
|
||||
assert.ok(fast);
|
||||
assert.deepEqual(fast.map((m) => m.ref), ['ready/ready-fast', 'nokey/nokey-model']);
|
||||
});
|
||||
|
||||
it('orders providers by priority', () => {
|
||||
const reg = make(providerDir(CATALOG), { READY_KEY: 'k1234567890abc', NOKEY_KEY: 'k1234567890abc' });
|
||||
assert.deepEqual(reg.all().map((p) => p.id), ['ready', 'nokey']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Registry — shipped provider files', () => {
|
||||
const dir = join(import.meta.dirname, '..', '..', '..', 'providers');
|
||||
|
||||
it('loads the real catalog with no errors', () => {
|
||||
const reg = new Registry({ dir, env: {}, logger: silentLogger });
|
||||
const errors = reg.issues().filter((i) => i.level === 'error');
|
||||
assert.deepEqual(errors, [], 'every shipped provider file must parse');
|
||||
assert.ok(reg.all().length >= 2);
|
||||
});
|
||||
|
||||
it('renders unconfigured providers as advice, not as a crash', () => {
|
||||
const reg = new Registry({ dir, env: {}, logger: silentLogger });
|
||||
const mistral = reg.get('mistral');
|
||||
assert.ok(mistral);
|
||||
assert.equal(mistral.configured, false);
|
||||
assert.equal(mistral.configError, 'no API key found (set one of: MISTRAL_API_KEY, MISTRAL_API_KEY_2)');
|
||||
|
||||
const cf = reg.get('cloudflare');
|
||||
assert.ok(cf);
|
||||
assert.equal(cf.configError, 'missing environment variable: CLOUDFLARE_ACCOUNT_ID');
|
||||
});
|
||||
|
||||
it('lights cloudflare up once both variables are present', () => {
|
||||
const reg = new Registry({
|
||||
dir,
|
||||
env: { CLOUDFLARE_ACCOUNT_ID: 'acct-1', CLOUDFLARE_API_TOKEN: 'cf-token-abcdefgh' },
|
||||
logger: silentLogger,
|
||||
});
|
||||
const cf = reg.get('cloudflare');
|
||||
assert.ok(cf);
|
||||
assert.equal(cf.configured, true);
|
||||
assert.equal(cf.baseUrl, 'https://api.cloudflare.com/client/v4/accounts/acct-1/ai/v1');
|
||||
assert.ok(cf.models.some((m) => m.spec.id.includes('/')), 'cloudflare model ids contain slashes');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user