Freeway: self-hosted OpenAI-compatible gateway for free-tier providers

One endpoint in front of every free LLM tier, with automatic failover and no
provider key ever leaving the box.

Providers are data, not code: adding one is a JSON file plus an env var, never
a TypeScript change. Ships 12 providers pre-written; `verify-provider --apply`
derives the live model list from your own key.

- core: registry, bucketed quota meter (rolling + UTC-calendar windows),
  key-pool store, 6-step model router with diagnostic rejection reasons
- gateway: node:http, failover that commits to a stream only after 200 headers,
  SSE passthrough with usage sniffing, virtual keys with admin/inference roles
- context engine: sqlite sessions and a 5-rung refit ladder, so a 128k thread
  keeps working when it fails over to an 8k model
- cache: exact (sha256, safe-by-default) + opt-in semantic (flat Float32Array)
- guard: inbound limits, quota forecasting, limit calibration from upstream
  headers, key health, secret/PII/injection scanning
- dashboard: one self-contained HTML file, live SSE request log, token usage
- sdk: zero-dep typed client, drop-in for the OpenAI SDK

Zero runtime dependencies. 326 tests, 27-check smoke suite against a scripted
mock upstream so routing is testable without spending quota.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

packages/core/test/store.test.ts

@@ -0,0 +1,235 @@
+import { strict as assert } from 'node:assert';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { after, describe, it } from 'node:test';
+
+import { Store, classifyFailure } from '../src/store.ts';
+import { silentLogger } from '../src/util.ts';
+
+const dirs: string[] = [];
+after(() => {
+  for (const d of dirs) rmSync(d, { recursive: true, force: true });
+});
+
+function tmp(): string {
+  const d = mkdtempSync(join(tmpdir(), 'freeway-store-'));
+  dirs.push(d);
+  return d;
+}
+
+describe('classifyFailure', () => {
+  it('blames the key for credential and rate-limit rejections', () => {
+    // Another key in the pool may well succeed, so only that key is benched.
+    for (const s of [401, 403, 429]) assert.equal(classifyFailure(s), 'key', `status ${s}`);
+  });
+
+  it('blames the provider for server and transport failures', () => {
+    for (const s of [500, 502, 503, 504, 408]) assert.equal(classifyFailure(s), 'provider', `status ${s}`);
+    assert.equal(classifyFailure(null), 'provider', 'network error');
+  });
+
+  it('blames neither for a malformed request', () => {
+    // A 400 will fail identically everywhere; retrying it is pure waste.
+    for (const s of [400, 404, 422]) assert.equal(classifyFailure(s), 'request', `status ${s}`);
+  });
+});
+
+describe('Store — failure attribution', () => {
+  it('a 429 benches one key and leaves the pool usable', () => {
+    const store = new Store({ file: null, cooldownMs: 30_000, logger: silentLogger });
+    const kind = store.recordFailure('mistral', 'mistral#0', { status: 429, message: 'rate limited' });
+
+    assert.equal(kind, 'key');
+    assert.equal(store.isKeyCooling('mistral', 'mistral#0'), true);
+    assert.equal(store.isKeyCooling('mistral', 'mistral#1'), false, 'the other key is untouched');
+    assert.equal(store.isProviderCooling('mistral'), false, 'the provider is fine');
+    assert.equal(store.key('mistral', 'mistral#0').status, 'rate-limited');
+  });
+
+  it('a 503 benches the whole provider', () => {
+    const store = new Store({ file: null, cooldownMs: 30_000, logger: silentLogger });
+    const kind = store.recordFailure('groq', 'groq#0', { status: 503, message: '503 from upstream' });
+
+    assert.equal(kind, 'provider');
+    assert.equal(store.isProviderCooling('groq'), true);
+    assert.equal(store.isKeyCooling('groq', 'groq#0'), false, 'the key is not at fault');
+    assert.equal(store.provider('groq').lastError, '503 from upstream');
+  });
+
+  it('a 400 benches nothing at all', () => {
+    const store = new Store({ file: null, logger: silentLogger });
+    const kind = store.recordFailure('mistral', 'mistral#0', { status: 400, message: 'bad request' });
+
+    assert.equal(kind, 'request');
+    assert.equal(store.isProviderCooling('mistral'), false);
+    assert.equal(store.isKeyCooling('mistral', 'mistral#0'), false);
+    assert.equal(store.provider('mistral').fail, 1, 'still counted as a failure');
+  });
+
+  it('benches a rejected credential for far longer than a rate limit', () => {
+    const store = new Store({ file: null, cooldownMs: 1000, logger: silentLogger });
+    store.recordFailure('p', 'p#0', { status: 401, message: 'unauthorized' });
+    store.recordFailure('p', 'p#1', { status: 429, message: 'slow down' });
+
+    const invalid = store.key('p', 'p#0');
+    const limited = store.key('p', 'p#1');
+    assert.equal(invalid.status, 'invalid');
+    assert.equal(limited.status, 'rate-limited');
+    assert.ok(invalid.cooldownUntil > limited.cooldownUntil, 'a bad key stays benched much longer');
+  });
+
+  it('honours an upstream Retry-After over the configured cooldown', () => {
+    const now = Date.UTC(2026, 0, 1);
+    const store = new Store({ file: null, cooldownMs: 60_000, clock: () => now, logger: silentLogger });
+    store.recordFailure('p', 'p#0', { status: 429, message: 'rate limited', retryAfterMs: 5_000 });
+    assert.equal(store.key('p', 'p#0').cooldownUntil, now + 5_000);
+  });
+
+  it('a success clears the provider bench and revives the key', () => {
+    const store = new Store({ file: null, logger: silentLogger });
+    store.recordFailure('p', 'p#0', { status: 502, message: 'bad gateway' });
+    assert.equal(store.isProviderCooling('p'), true);
+
+    store.recordSuccess('p', 'p#0', 120);
+    assert.equal(store.isProviderCooling('p'), false);
+    assert.equal(store.key('p', 'p#0').status, 'ok');
+    assert.equal(store.provider('p').lastError, null);
+  });
+});
+
+describe('Store — accounting', () => {
+  it('tracks EWMA latency rather than the last value', () => {
+    const store = new Store({ file: null, logger: silentLogger });
+    store.recordSuccess('p', 'p#0', 100);
+    assert.equal(store.provider('p').latencyMs, 100, 'seeds on the first sample');
+
+    store.recordSuccess('p', 'p#0', 200);
+    const l = store.provider('p').latencyMs ?? 0;
+    assert.ok(l > 100 && l < 200, `smoothed toward the new sample, got ${l}`);
+  });
+
+  it('meters both the provider and the individual key', () => {
+    const store = new Store({ file: null, logger: silentLogger });
+    store.recordSuccess('p', 'p#0', 10, 500);
+    store.recordSuccess('p', 'p#1', 10, 300);
+
+    assert.equal(store.provider('p').meter.used('tpm'), 800);
+    assert.equal(store.key('p', 'p#0').meter.used('tpm'), 500);
+    assert.equal(store.key('p', 'p#1').meter.used('tpm'), 300);
+  });
+
+  it('lands late stream tokens on both meters', () => {
+    const store = new Store({ file: null, logger: silentLogger });
+    store.recordSuccess('p', 'p#0', 10, 0);
+    store.addTokens('p', 'p#0', 750);
+
+    assert.equal(store.provider('p').meter.used('tpm'), 750);
+    assert.equal(store.key('p', 'p#0').meter.used('tpm'), 750);
+  });
+
+  it('rotates the round-robin cursor', () => {
+    const store = new Store({ file: null, logger: silentLogger });
+    assert.equal(store.nextCursor('p'), 1);
+    assert.equal(store.nextCursor('p'), 2);
+    assert.equal(store.nextCursor('other'), 1, 'cursors are per provider');
+  });
+});
+
+describe('Store — request log', () => {
+  const entry = (i: number) => ({
+    id: `r${i}`,
+    at: i,
+    requestedModel: 'auto',
+    resolution: 'auto',
+    providerId: 'p',
+    modelId: 'm',
+    keyId: 'p#0',
+    attempts: [],
+    status: 200,
+    ms: 10,
+    tokensIn: 1,
+    tokensOut: 1,
+    stream: false,
+    cache: null,
+    error: null,
+  });
+
+  it('returns entries newest first', () => {
+    const store = new Store({ file: null, logSize: 10, logger: silentLogger });
+    for (let i = 0; i < 3; i++) store.addLog(entry(i));
+    assert.deepEqual(store.logs().map((e) => e.id), ['r2', 'r1', 'r0']);
+  });
+
+  it('is a ring buffer that never grows past its size', () => {
+    const store = new Store({ file: null, logSize: 5, logger: silentLogger });
+    for (let i = 0; i < 100; i++) store.addLog(entry(i));
+
+    const logs = store.logs(1000);
+    assert.equal(logs.length, 5, 'memory stays flat under sustained traffic');
+    assert.deepEqual(logs.map((e) => e.id), ['r99', 'r98', 'r97', 'r96', 'r95']);
+  });
+
+  it('issues monotonic request ids', () => {
+    const store = new Store({ file: null, logger: silentLogger });
+    const a = store.nextRequestId();
+    const b = store.nextRequestId();
+    assert.notEqual(a, b);
+  });
+});
+
+describe('Store — persistence', () => {
+  it('round-trips meters and counts through a restart', () => {
+    const dir = tmp();
+    const file = join(dir, 'usage.json');
+
+    const first = new Store({ file, logger: silentLogger });
+    first.recordSuccess('mistral', 'mistral#0', 120, 4000);
+    first.recordFailure('mistral', 'mistral#0', { status: 429, message: 'rate limited' });
+    first.save();
+
+    const second = new Store({ file, logger: silentLogger });
+    assert.equal(second.provider('mistral').ok, 1);
+    assert.equal(second.provider('mistral').fail, 1);
+    // The whole point: a monthly counter must not silently reset on restart.
+    assert.equal(second.provider('mistral').meter.used('tpmo'), 4000);
+    assert.equal(second.key('mistral', 'mistral#0').meter.used('tpmo'), 4000);
+  });
+
+  it('writes atomically and leaves no temp files behind', () => {
+    const dir = tmp();
+    const file = join(dir, 'usage.json');
+    const store = new Store({ file, logger: silentLogger });
+    store.recordSuccess('p', 'p#0', 10, 5);
+    store.save();
+
+    const parsed: unknown = JSON.parse(readFileSync(file, 'utf8'));
+    assert.equal((parsed as { version: number }).version, 1);
+  });
+
+  it('ignores a corrupt usage file instead of refusing to start', () => {
+    const dir = tmp();
+    const file = join(dir, 'usage.json');
+    writeFileSync(file, '{ not json');
+
+    let store: Store | undefined;
+    assert.doesNotThrow(() => {
+      store = new Store({ file, logger: silentLogger });
+    });
+    assert.equal(store?.provider('p').ok, 0);
+  });
+
+  it('ignores a usage file from a future version', () => {
+    const dir = tmp();
+    const file = join(dir, 'usage.json');
+    writeFileSync(file, JSON.stringify({ version: 999, savedAt: 0, providers: { p: { ok: 5 } } }));
+
+    const store = new Store({ file, logger: silentLogger });
+    assert.equal(store.provider('p').ok, 0, 'unknown schema is discarded, not half-read');
+  });
+
+  it('does nothing when persistence is disabled', () => {
+    const store = new Store({ file: null, logger: silentLogger });
+    assert.doesNotThrow(() => store.save());
+  });
+});