Freeway: self-hosted OpenAI-compatible gateway for free-tier providers
One endpoint in front of every free LLM tier, with automatic failover and no provider key ever leaving the box. Providers are data, not code: adding one is a JSON file plus an env var, never a TypeScript change. Ships 12 providers pre-written; `verify-provider --apply` derives the live model list from your own key. - core: registry, bucketed quota meter (rolling + UTC-calendar windows), key-pool store, 6-step model router with diagnostic rejection reasons - gateway: node:http, failover that commits to a stream only after 200 headers, SSE passthrough with usage sniffing, virtual keys with admin/inference roles - context engine: sqlite sessions and a 5-rung refit ladder, so a 128k thread keeps working when it fails over to an 8k model - cache: exact (sha256, safe-by-default) + opt-in semantic (flat Float32Array) - guard: inbound limits, quota forecasting, limit calibration from upstream headers, key health, secret/PII/injection scanning - dashboard: one self-contained HTML file, live SSE request log, token usage - sdk: zero-dep typed client, drop-in for the OpenAI SDK Zero runtime dependencies. 326 tests, 27-check smoke suite against a scripted mock upstream so routing is testable without spending quota. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Aryan SinghandClaude Opus 5 committed
commit
cb985dcf9c
86 files changed
+16350
No files matched your search
@@ -0,0 +1,310 @@
|
||||
/**
|
||||
* A fake OpenAI-compatible provider you control completely.
|
||||
*
|
||||
* This is the most valuable thing in the repo: it means routing, failover,
|
||||
* streaming, quota and key-pool behaviour can all be tested without spending a
|
||||
* single token of real free-tier quota. Every test from Phase 3 onward points
|
||||
* at one of these instead of a real provider.
|
||||
*
|
||||
* Usage as a library:
|
||||
* const mock = await createMockUpstream({ name: 'alpha' });
|
||||
* mock.script([{ status: 429, headers: { 'retry-after': '2' } }, { status: 200 }]);
|
||||
* // ... point a provider's baseUrl at mock.url ...
|
||||
* mock.close();
|
||||
*
|
||||
* Usage as a standalone server (for `npm run smoke`):
|
||||
* node scripts/mock-upstream.js --port 9001 --name alpha
|
||||
*/
|
||||
|
||||
import { createServer } from 'node:http';
|
||||
import { once } from 'node:events';
|
||||
|
||||
/**
|
||||
* @typedef {object} ScriptStep
|
||||
* @property {number} [status] HTTP status to return. Default 200.
|
||||
* @property {Record<string,string>} [headers] Extra response headers (rate-limit headers go here).
|
||||
* @property {unknown} [body] Body override. Default: a plausible OpenAI response.
|
||||
* @property {number} [delayMs] Artificial latency before responding.
|
||||
* @property {boolean} [dropUsage] Omit the `usage` block, to exercise the estimator fallback.
|
||||
* @property {boolean} [hang] Never respond, to exercise client timeouts.
|
||||
* @property {string} [content] Assistant text to return.
|
||||
* @property {number} [repeat] How many requests this step covers. Default 1; Infinity for the tail.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {object} MockOptions
|
||||
* @property {string} [name] Shows up in responses and logs, so failover is legible.
|
||||
* @property {number} [port] 0 picks a free port.
|
||||
* @property {string} [host]
|
||||
* @property {ScriptStep[]} [script] Consumed in order; the last step repeats forever.
|
||||
* @property {Record<string, ScriptStep>} [keys] Per-API-key overrides, keyed by the bearer token.
|
||||
* @property {string[]} [models] Ids returned from GET /models.
|
||||
* @property {number} [latencyMs] Baseline latency for every response.
|
||||
*/
|
||||
|
||||
const DEFAULT_MODELS = ['mock-small', 'mock-large', '@mock/vendor/slashy-model'];
|
||||
|
||||
export async function createMockUpstream(options = {}) {
|
||||
const name = options.name ?? 'mock';
|
||||
const baseLatency = options.latencyMs ?? 0;
|
||||
|
||||
/** @type {ScriptStep[]} */
|
||||
let script = options.script ? [...options.script] : [];
|
||||
let stepIndex = 0;
|
||||
let stepUses = 0;
|
||||
|
||||
/** @type {Record<string, ScriptStep>} */
|
||||
let keyOverrides = { ...(options.keys ?? {}) };
|
||||
let models = options.models ? [...options.models] : [...DEFAULT_MODELS];
|
||||
|
||||
/** @type {{ path: string, method: string, body: any, key: string|null, headers: Record<string,string>, at: number }[]} */
|
||||
const calls = [];
|
||||
|
||||
/** Pull the next scripted step, honouring `repeat`. The final step sticks. */
|
||||
function nextStep() {
|
||||
if (script.length === 0) return {};
|
||||
const step = script[Math.min(stepIndex, script.length - 1)];
|
||||
const repeat = step.repeat ?? 1;
|
||||
stepUses += 1;
|
||||
if (stepUses >= repeat && stepIndex < script.length - 1) {
|
||||
stepIndex += 1;
|
||||
stepUses = 0;
|
||||
}
|
||||
return step;
|
||||
}
|
||||
|
||||
const server = createServer((req, res) => {
|
||||
let raw = '';
|
||||
req.on('data', (c) => {
|
||||
raw += c;
|
||||
});
|
||||
req.on('end', async () => {
|
||||
const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
|
||||
const path = url.pathname.replace(/\/v1(?=\/|$)/, '') || '/';
|
||||
const auth = req.headers.authorization;
|
||||
const key = typeof auth === 'string' ? auth.replace(/^Bearer\s+/i, '') : null;
|
||||
|
||||
let body = null;
|
||||
if (raw) {
|
||||
try {
|
||||
body = JSON.parse(raw);
|
||||
} catch {
|
||||
body = raw;
|
||||
}
|
||||
}
|
||||
|
||||
calls.push({
|
||||
path,
|
||||
method: req.method ?? 'GET',
|
||||
body,
|
||||
key,
|
||||
headers: /** @type {Record<string,string>} */ (Object.fromEntries(Object.entries(req.headers).map(([k, v]) => [k, String(v)]))),
|
||||
at: Date.now(),
|
||||
});
|
||||
|
||||
// A control plane, so a running mock can be re-scripted mid-test.
|
||||
if (path === '/__control' && req.method === 'POST') {
|
||||
if (body && typeof body === 'object') {
|
||||
if (Array.isArray(body.script)) {
|
||||
script = body.script;
|
||||
stepIndex = 0;
|
||||
stepUses = 0;
|
||||
}
|
||||
if (body.keys) keyOverrides = body.keys;
|
||||
if (Array.isArray(body.models)) models = body.models;
|
||||
if (body.reset) calls.length = 0;
|
||||
}
|
||||
return send(res, 200, {}, { ok: true });
|
||||
}
|
||||
|
||||
if (path === '/__calls') return send(res, 200, {}, { calls });
|
||||
|
||||
// A per-key override wins over the script — that is how "one key in the
|
||||
// pool is revoked while the others work" gets tested.
|
||||
const step = (key !== null && keyOverrides[key]) || nextStep();
|
||||
|
||||
const delay = step.delayMs ?? baseLatency;
|
||||
if (delay > 0) await sleep(delay);
|
||||
if (step.hang) return; // never responds; the client's AbortController must fire
|
||||
|
||||
const status = step.status ?? 200;
|
||||
const headers = { ...(step.headers ?? {}) };
|
||||
|
||||
if (path === '/models' && req.method === 'GET') {
|
||||
if (status !== 200) return send(res, status, headers, step.body ?? errorBody(status, name));
|
||||
return send(res, 200, headers, { object: 'list', data: models.map((id) => ({ id, object: 'model', owned_by: name })) });
|
||||
}
|
||||
|
||||
if (status !== 200) {
|
||||
return send(res, status, headers, step.body ?? errorBody(status, name));
|
||||
}
|
||||
|
||||
if (path === '/embeddings') {
|
||||
const input = Array.isArray(body?.input) ? body.input : [body?.input ?? ''];
|
||||
return send(res, 200, headers, {
|
||||
object: 'list',
|
||||
model: body?.model ?? 'mock-embed',
|
||||
data: input.map((text, i) => ({ object: 'embedding', index: i, embedding: fakeEmbedding(String(text)) })),
|
||||
usage: step.dropUsage ? undefined : { prompt_tokens: estimate(input.join(' ')), total_tokens: estimate(input.join(' ')) },
|
||||
});
|
||||
}
|
||||
|
||||
if (path === '/chat/completions') {
|
||||
const prompt = messagesToText(body?.messages);
|
||||
const content = step.content ?? `[${name}] ${prompt.slice(-120)}`;
|
||||
const usage = step.dropUsage
|
||||
? undefined
|
||||
: { prompt_tokens: estimate(prompt), completion_tokens: estimate(content), total_tokens: estimate(prompt) + estimate(content) };
|
||||
|
||||
if (body?.stream) return streamChat(res, { name, model: body?.model ?? 'mock', content, usage, headers });
|
||||
return send(res, 200, headers, {
|
||||
id: `chatcmpl-${name}-${calls.length}`,
|
||||
object: 'chat.completion',
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
model: body?.model ?? 'mock',
|
||||
choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],
|
||||
usage,
|
||||
});
|
||||
}
|
||||
|
||||
return send(res, 404, {}, { error: { message: `mock upstream has no route ${path}`, type: 'invalid_request_error' } });
|
||||
});
|
||||
});
|
||||
|
||||
server.listen(options.port ?? 0, options.host ?? '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const address = server.address();
|
||||
const port = typeof address === 'object' && address !== null ? address.port : 0;
|
||||
|
||||
return {
|
||||
name,
|
||||
port,
|
||||
url: `http://127.0.0.1:${port}/v1`,
|
||||
calls,
|
||||
/** Replace the script and rewind. */
|
||||
script(steps) {
|
||||
script = [...steps];
|
||||
stepIndex = 0;
|
||||
stepUses = 0;
|
||||
},
|
||||
setKeys(k) {
|
||||
keyOverrides = { ...k };
|
||||
},
|
||||
setModels(m) {
|
||||
models = [...m];
|
||||
},
|
||||
reset() {
|
||||
calls.length = 0;
|
||||
stepIndex = 0;
|
||||
stepUses = 0;
|
||||
},
|
||||
/** Requests seen on a given path. */
|
||||
callsTo(path) {
|
||||
return calls.filter((c) => c.path === path);
|
||||
},
|
||||
close() {
|
||||
return new Promise((resolve) => server.close(() => resolve(undefined)));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function send(res, status, headers, body) {
|
||||
const payload = JSON.stringify(body ?? {});
|
||||
res.writeHead(status, {
|
||||
'content-type': 'application/json',
|
||||
'content-length': Buffer.byteLength(payload),
|
||||
...headers,
|
||||
});
|
||||
res.end(payload);
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit an SSE stream shaped like OpenAI's, including the trailing usage-only
|
||||
* chunk that real providers send last. Token accounting depends on finding it.
|
||||
*/
|
||||
function streamChat(res, { name, model, content, usage, headers }) {
|
||||
res.writeHead(200, {
|
||||
'content-type': 'text/event-stream',
|
||||
'cache-control': 'no-cache',
|
||||
connection: 'keep-alive',
|
||||
...headers,
|
||||
});
|
||||
|
||||
const id = `chatcmpl-${name}-stream`;
|
||||
const created = Math.floor(Date.now() / 1000);
|
||||
const frame = (delta, finish = null) =>
|
||||
`data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, model, choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`;
|
||||
|
||||
res.write(frame({ role: 'assistant' }));
|
||||
// Deliberately chunked mid-word so the gateway's line buffering is exercised.
|
||||
for (const piece of chunkString(content, 7)) res.write(frame({ content: piece }));
|
||||
res.write(frame({}, 'stop'));
|
||||
|
||||
if (usage) {
|
||||
res.write(`data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, model, choices: [], usage })}\n\n`);
|
||||
}
|
||||
res.write('data: [DONE]\n\n');
|
||||
res.end();
|
||||
}
|
||||
|
||||
function chunkString(s, size) {
|
||||
const out = [];
|
||||
for (let i = 0; i < s.length; i += size) out.push(s.slice(i, i + size));
|
||||
return out.length > 0 ? out : [''];
|
||||
}
|
||||
|
||||
function messagesToText(messages) {
|
||||
if (!Array.isArray(messages)) return '';
|
||||
return messages
|
||||
.map((m) => {
|
||||
if (typeof m?.content === 'string') return m.content;
|
||||
if (Array.isArray(m?.content)) return m.content.map((p) => (typeof p?.text === 'string' ? p.text : '')).join(' ');
|
||||
return '';
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function estimate(text) {
|
||||
return Math.max(1, Math.ceil(String(text).length / 4));
|
||||
}
|
||||
|
||||
/** Deterministic pseudo-embedding, so semantic-cache tests are reproducible. */
|
||||
function fakeEmbedding(text, dims = 8) {
|
||||
const v = new Array(dims).fill(0);
|
||||
for (let i = 0; i < text.length; i++) v[i % dims] += text.charCodeAt(i);
|
||||
const norm = Math.hypot(...v) || 1;
|
||||
return v.map((x) => x / norm);
|
||||
}
|
||||
|
||||
function errorBody(status, name) {
|
||||
const type = status === 429 ? 'rate_limit_exceeded' : status >= 500 ? 'server_error' : 'invalid_request_error';
|
||||
return { error: { message: `[${name}] mock upstream returned ${status}`, type, code: status } };
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Standalone mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
const args = process.argv.slice(2);
|
||||
const arg = (flag, fallback) => {
|
||||
const i = args.indexOf(flag);
|
||||
return i >= 0 && args[i + 1] !== undefined ? args[i + 1] : fallback;
|
||||
};
|
||||
const mock = await createMockUpstream({
|
||||
name: arg('--name', 'mock'),
|
||||
port: Number(arg('--port', '0')),
|
||||
latencyMs: Number(arg('--latency', '0')),
|
||||
});
|
||||
console.log(`mock upstream "${mock.name}" listening on ${mock.url}`);
|
||||
console.log(` control: POST ${mock.url.replace('/v1', '')}/__control {"script":[{"status":429}]}`);
|
||||
process.on('SIGINT', () => {
|
||||
mock.close().then(() => process.exit(0));
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Scaffold a new provider file.
|
||||
*
|
||||
* node scripts/new-provider.mjs groq --url https://api.groq.com/openai/v1 --env GROQ_API_KEY
|
||||
*
|
||||
* Writes providers/<id>.json with every field present and commented via the
|
||||
* `notes` field, then tells you to run the verifier. Nothing here touches
|
||||
* TypeScript, which is the whole point.
|
||||
*/
|
||||
|
||||
import { existsSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const id = args.find((a) => !a.startsWith('--'));
|
||||
|
||||
if (!id || args.includes('--help') || args.includes('-h')) {
|
||||
console.log(`
|
||||
Usage: node scripts/new-provider.mjs <id> [options]
|
||||
|
||||
<id> short lowercase slug, e.g. "groq"
|
||||
--url <baseUrl> OpenAI-compatible base URL
|
||||
--env <VAR> env var holding the key (default: <ID>_API_KEY)
|
||||
--auth <type> bearer | header | query | none (default: bearer)
|
||||
--header <name> header name when --auth header
|
||||
--label <name> display name
|
||||
--force overwrite an existing file
|
||||
|
||||
Then:
|
||||
node scripts/verify-provider.mjs ${id ?? '<id>'} # check it against the live API
|
||||
`);
|
||||
process.exit(id ? 0 : 1);
|
||||
}
|
||||
|
||||
if (!/^[a-z0-9][a-z0-9-]*$/.test(id)) {
|
||||
console.error(`error: id must be lowercase letters, digits and dashes — got "${id}"`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const flag = (name, fallback) => {
|
||||
const i = args.indexOf(`--${name}`);
|
||||
return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : fallback;
|
||||
};
|
||||
|
||||
const envVar = flag('env', `${id.toUpperCase().replace(/-/g, '_')}_API_KEY`);
|
||||
const authType = flag('auth', 'bearer');
|
||||
const outPath = join('providers', `${id}.json`);
|
||||
|
||||
if (existsSync(outPath) && !args.includes('--force')) {
|
||||
console.error(`error: ${outPath} already exists (use --force to overwrite)`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const spec = {
|
||||
$schema: './_schema.json',
|
||||
id,
|
||||
label: flag('label', id.replace(/(^|-)(\w)/g, (_, s, c) => (s ? ' ' : '') + c.toUpperCase())),
|
||||
docs: null,
|
||||
console: null,
|
||||
enabled: true,
|
||||
priority: 50,
|
||||
adapter: 'openai',
|
||||
baseUrl: flag('url', 'https://api.example.com/v1'),
|
||||
auth: {
|
||||
type: authType,
|
||||
...(authType === 'header' ? { header: flag('header', 'x-api-key') } : {}),
|
||||
...(authType === 'query' ? { query: flag('query', 'key') } : {}),
|
||||
envKeys: authType === 'none' ? [] : [envVar],
|
||||
},
|
||||
// Leave these null until verified. A null is honest and the guard layer fills
|
||||
// it in from real response headers; a guessed number silently misroutes.
|
||||
limits: { rps: null, rpm: null, rpd: null, tpm: null, tpd: null, tpmo: null },
|
||||
limitsSource: 'unverified',
|
||||
verifiedOn: new Date().toISOString().slice(0, 10),
|
||||
modelsEndpoint: '/models',
|
||||
modelsPath: 'data[].id',
|
||||
dropParams: [],
|
||||
headers: {},
|
||||
notes: `TODO: signup friction, quirks, and why any number above is what it is. Run scripts/verify-provider.mjs ${id} to discover models and observe real limits.`,
|
||||
models: [],
|
||||
};
|
||||
|
||||
writeFileSync(outPath, `${JSON.stringify(spec, null, 2)}\n`);
|
||||
|
||||
console.log(`created ${outPath}`);
|
||||
console.log(`
|
||||
next:
|
||||
1. export ${envVar}=<your key>
|
||||
2. node scripts/verify-provider.mjs ${id}
|
||||
3. paste the models the verifier found into "models", then re-run it
|
||||
`);
|
||||
@@ -0,0 +1,373 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Exercise every route against mock upstreams. No real quota is spent.
|
||||
*
|
||||
* npm run smoke
|
||||
*
|
||||
* This is the thing to run after adding a provider or touching routing. It
|
||||
* boots two fake providers and a real gateway, then walks the whole surface —
|
||||
* routing, failover, streaming, sessions, cache, guard, probing, dashboard API.
|
||||
*
|
||||
* Exit code 0 if everything passed.
|
||||
*/
|
||||
|
||||
import { once } from 'node:events';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import {
|
||||
ContextStore, ExactCache, Registry, Router, SemanticCache, Store, Tokenizer,
|
||||
parseConfig, silentLogger,
|
||||
} from '../packages/core/src/index.ts';
|
||||
import { createGateway } from '../packages/gateway/src/server.ts';
|
||||
import { createMockUpstream } from './mock-upstream.js';
|
||||
|
||||
const C = process.stdout.isTTY
|
||||
? { g: '\x1b[32m', r: '\x1b[31m', d: '\x1b[2m', B: '\x1b[1m', x: '\x1b[0m' }
|
||||
: { g: '', r: '', d: '', B: '', x: '' };
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
const failures = [];
|
||||
|
||||
async function check(name, fn) {
|
||||
try {
|
||||
const detail = await fn();
|
||||
passed += 1;
|
||||
console.log(` ${C.g}✓${C.x} ${name}${detail ? ` ${C.d}${detail}${C.x}` : ''}`);
|
||||
} catch (err) {
|
||||
failed += 1;
|
||||
failures.push({ name, message: err instanceof Error ? err.message : String(err) });
|
||||
console.log(` ${C.r}✗${C.x} ${name}\n ${C.r}${err instanceof Error ? err.message : err}${C.x}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assert(condition, message) {
|
||||
if (!condition) throw new Error(message);
|
||||
}
|
||||
|
||||
function eq(actual, expected, what) {
|
||||
if (actual !== expected) throw new Error(`${what}: expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const alpha = await createMockUpstream({ name: 'alpha' });
|
||||
const beta = await createMockUpstream({ name: 'beta' });
|
||||
|
||||
const dir = mkdtempSync(join(tmpdir(), 'freeway-smoke-'));
|
||||
writeFileSync(join(dir, 'alpha.json'), JSON.stringify({
|
||||
id: 'alpha', label: 'Alpha (mock)', priority: 10, baseUrl: alpha.url,
|
||||
auth: { type: 'bearer', envKeys: ['ALPHA_KEY'] }, limits: { rpm: 500 },
|
||||
models: [
|
||||
{ id: 'alpha-chat', alias: ['fast', 'chat'], context: 128000, caps: ['chat', 'tools', 'json'] },
|
||||
{ id: 'alpha-embed', alias: ['embed'], context: 8000, caps: ['embed'] },
|
||||
],
|
||||
}));
|
||||
writeFileSync(join(dir, 'beta.json'), JSON.stringify({
|
||||
id: 'beta', label: 'Beta (mock)', priority: 20, baseUrl: beta.url,
|
||||
auth: { type: 'bearer', envKeys: ['BETA_KEY'] }, limits: { rpm: 500 },
|
||||
models: [{ id: '@cf/meta/llama-3.1-8b-instruct', alias: ['fast'], context: 8192, caps: ['chat'] }],
|
||||
}));
|
||||
|
||||
const registry = new Registry({ dir, env: { ALPHA_KEY: 'ak-smoke-0001,ak-smoke-0002', BETA_KEY: 'bk-smoke-0001' }, logger: silentLogger });
|
||||
const store = new Store({ file: null, logger: silentLogger });
|
||||
const { config } = parseConfig({}, {});
|
||||
config.cache.mode = 'safe';
|
||||
const router = new Router(registry, store, config);
|
||||
const context = new ContextStore(':memory:');
|
||||
const cache = new ExactCache({ file: ':memory:', mode: 'safe', ttlMs: 3_600_000, maxEntries: 100 });
|
||||
|
||||
const server = createGateway({
|
||||
registry, store, router, config, logger: silentLogger,
|
||||
contextStore: context, tokenizer: new Tokenizer(),
|
||||
cache, semanticCache: new SemanticCache({ enabled: false, threshold: 0.92, maxEntries: 10 }),
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const B = `http://127.0.0.1:${server.address().port}`;
|
||||
|
||||
const post = (path, body, headers = {}) =>
|
||||
fetch(`${B}${path}`, { method: 'POST', headers: { 'content-type': 'application/json', ...headers }, body: JSON.stringify(body) });
|
||||
const chat = (over = {}) => post('/v1/chat/completions', { model: 'auto', messages: [{ role: 'user', content: 'hello' }], ...over });
|
||||
|
||||
console.log(`\n${C.B}Freeway smoke test${C.x} ${C.d}${B}${C.x}\n`);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log(`${C.B}core routes${C.x}`);
|
||||
|
||||
await check('GET /healthz', async () => {
|
||||
const body = await (await fetch(`${B}/healthz`)).json();
|
||||
eq(body.ok, true, 'ok');
|
||||
eq(body.providers.usable, 2, 'usable providers');
|
||||
return `${body.providers.usable}/${body.providers.total} providers usable`;
|
||||
});
|
||||
|
||||
await check('GET / serves the dashboard', async () => {
|
||||
const res = await fetch(`${B}/`);
|
||||
eq(res.status, 200, 'status');
|
||||
const html = await res.text();
|
||||
assert(html.includes('<title>Freeway</title>'), 'no title');
|
||||
assert(!/\b(?:src|href)\s*=\s*["'](?:https?:)?\/\//i.test(html), 'dashboard references an external asset');
|
||||
return 'self-contained, no CDN';
|
||||
});
|
||||
|
||||
await check('GET /v1/models', async () => {
|
||||
const body = await (await fetch(`${B}/v1/models`)).json();
|
||||
const ids = body.data.map((d) => d.id);
|
||||
assert(ids.includes('alpha/alpha-chat'), 'missing provider/model id');
|
||||
assert(ids.includes('beta/@cf/meta/llama-3.1-8b-instruct'), 'slashy id mangled');
|
||||
assert(ids.includes('auto'), 'missing virtual model');
|
||||
return `${ids.length} entries`;
|
||||
});
|
||||
|
||||
await check('POST /v1/chat/completions', async () => {
|
||||
const res = await chat();
|
||||
eq(res.status, 200, 'status');
|
||||
eq(res.headers.get('x-freeway-provider'), 'alpha', 'provider');
|
||||
const body = await res.json();
|
||||
assert(body.choices[0].message.content.includes('[alpha]'), 'wrong upstream answered');
|
||||
return `served by ${res.headers.get('x-freeway-provider')} in ${res.headers.get('x-freeway-ms')}ms`;
|
||||
});
|
||||
|
||||
await check('POST /v1/embeddings', async () => {
|
||||
const res = await post('/v1/embeddings', { model: 'embed', input: ['a', 'b'] });
|
||||
eq(res.status, 200, 'status');
|
||||
eq(res.headers.get('x-freeway-model'), 'alpha-embed', 'model');
|
||||
const body = await res.json();
|
||||
eq(body.data.length, 2, 'vector count');
|
||||
return '2 vectors';
|
||||
});
|
||||
|
||||
await check('streaming', async () => {
|
||||
const res = await chat({ model: 'alpha/alpha-chat', stream: true });
|
||||
eq(res.status, 200, 'status');
|
||||
assert((res.headers.get('content-type') ?? '').includes('text/event-stream'), 'not SSE');
|
||||
const text = await res.text();
|
||||
assert(text.includes('data: [DONE]'), 'stream did not terminate');
|
||||
return 'SSE proxied intact';
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log(`\n${C.B}routing and failover${C.x}`);
|
||||
|
||||
await check('alias resolution', async () => {
|
||||
const res = await chat({ model: 'fast' });
|
||||
eq(res.headers.get('x-freeway-route'), 'alias:fast', 'resolution');
|
||||
return 'alias:fast';
|
||||
});
|
||||
|
||||
await check('provider/model pin survives slashes', async () => {
|
||||
const res = await chat({ model: 'beta/@cf/meta/llama-3.1-8b-instruct' });
|
||||
eq(res.status, 200, 'status');
|
||||
eq(res.headers.get('x-freeway-model'), '@cf/meta/llama-3.1-8b-instruct', 'model');
|
||||
return 'first-slash split';
|
||||
});
|
||||
|
||||
await check('429 on alpha fails over to beta', async () => {
|
||||
alpha.script([{ status: 429, repeat: 99 }]);
|
||||
const res = await chat({ model: 'fast' });
|
||||
eq(res.status, 200, 'status');
|
||||
eq(res.headers.get('x-freeway-provider'), 'beta', 'provider');
|
||||
alpha.script([]);
|
||||
store.provider('alpha').cooldownUntil = 0;
|
||||
for (const k of store.provider('alpha').keys.values()) k.cooldownUntil = 0;
|
||||
return `failed over after ${res.headers.get('x-freeway-attempts')} attempts`;
|
||||
});
|
||||
|
||||
await check('400 returns immediately without retrying', async () => {
|
||||
alpha.script([{ status: 400, body: { error: { message: 'bad param' } }, repeat: 1 }, { status: 200, repeat: 99 }]);
|
||||
const before = beta.callsTo('/chat/completions').length;
|
||||
const res = await chat({ model: 'alpha/alpha-chat' });
|
||||
eq(res.status, 400, 'status');
|
||||
eq(beta.callsTo('/chat/completions').length, before, 'beta should not have been tried');
|
||||
alpha.script([]);
|
||||
return 'no wasted retry';
|
||||
});
|
||||
|
||||
await check('unknown model returns 404 with suggestions', async () => {
|
||||
const res = await chat({ model: 'gpt-9-ultra' });
|
||||
eq(res.status, 404, 'status');
|
||||
const body = await res.json();
|
||||
eq(body.error.code, 'model_not_found', 'code');
|
||||
return `${body.freeway.suggestions.length} suggestions`;
|
||||
});
|
||||
|
||||
await check('exhausted quota returns 429 with diagnostics', async () => {
|
||||
const saved = registry.get('alpha').spec.limits.rpm;
|
||||
registry.get('alpha').spec.limits.rpm = 1;
|
||||
registry.get('beta').spec.limits.rpm = 1;
|
||||
for (let i = 0; i < 3; i++) { store.recordSuccess('alpha', 'alpha#0', 1); store.recordSuccess('beta', 'beta#0', 1); }
|
||||
|
||||
const res = await chat({ model: 'fast' });
|
||||
eq(res.status, 429, 'status');
|
||||
const body = await res.json();
|
||||
eq(body.error.code, 'all_providers_blocked', 'code');
|
||||
assert(body.freeway.blocked.length > 0, 'no blocked reasons');
|
||||
assert(/rpm limit reached/.test(body.freeway.blocked[0].reason), `reason not diagnostic: ${body.freeway.blocked[0].reason}`);
|
||||
|
||||
registry.get('alpha').spec.limits.rpm = saved;
|
||||
registry.get('beta').spec.limits.rpm = saved;
|
||||
return body.freeway.blocked[0].reason;
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log(`\n${C.B}context engine${C.x}`);
|
||||
|
||||
await check('session rebuilds the thread', async () => {
|
||||
await post('/v1/chat/completions', { model: 'fast', session: 'smoke', messages: [{ role: 'user', content: 'my name is Ada' }] });
|
||||
const res = await post('/v1/chat/completions', { model: 'fast', session: 'smoke', messages: [{ role: 'user', content: 'what is my name?' }] });
|
||||
eq(res.status, 200, 'status');
|
||||
eq(res.headers.get('x-freeway-session'), 'smoke', 'session header');
|
||||
const stored = context.messages('smoke');
|
||||
assert(stored.length >= 3, `expected stored history, got ${stored.length}`);
|
||||
return `${stored.length} messages stored`;
|
||||
});
|
||||
|
||||
await check('refit reshapes history for a smaller model', async () => {
|
||||
const turn = 'padding '.repeat(300);
|
||||
const messages = [{ role: 'system', content: 'be helpful' }];
|
||||
for (let i = 0; i < 20; i++) messages.push({ role: i % 2 ? 'assistant' : 'user', content: `${i}: ${turn}` });
|
||||
messages.push({ role: 'user', content: 'short question' });
|
||||
|
||||
alpha.script([{ status: 429, repeat: 99 }]);
|
||||
const res = await post('/v1/chat/completions', { model: 'fast', messages });
|
||||
eq(res.status, 200, 'status');
|
||||
eq(res.headers.get('x-freeway-provider'), 'beta', 'provider');
|
||||
const note = res.headers.get('x-freeway-context');
|
||||
assert(note, 'no x-freeway-context header');
|
||||
alpha.script([]);
|
||||
store.provider('alpha').cooldownUntil = 0;
|
||||
for (const k of store.provider('alpha').keys.values()) k.cooldownUntil = 0;
|
||||
return note;
|
||||
});
|
||||
|
||||
await check('GET /api/sessions', async () => {
|
||||
const body = await (await fetch(`${B}/api/sessions`)).json();
|
||||
assert(body.sessions.some((s) => s.id === 'smoke'), 'session missing');
|
||||
return `${body.sessions.length} sessions`;
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log(`\n${C.B}cache${C.x}`);
|
||||
|
||||
await check('identical request hits the cache', async () => {
|
||||
const q = { model: 'alpha/alpha-chat', temperature: 0, messages: [{ role: 'user', content: 'cache me' }] };
|
||||
await post('/v1/chat/completions', q);
|
||||
const before = alpha.callsTo('/chat/completions').length;
|
||||
const res = await post('/v1/chat/completions', q);
|
||||
eq(res.headers.get('x-freeway-cache'), 'exact', 'cache header');
|
||||
eq(alpha.callsTo('/chat/completions').length, before, 'upstream was called on a hit');
|
||||
return 'no upstream call, no quota';
|
||||
});
|
||||
|
||||
await check('GET /api/cache', async () => {
|
||||
const body = await (await fetch(`${B}/api/cache`)).json();
|
||||
eq(body.enabled, true, 'enabled');
|
||||
assert(body.exact.hits >= 1, 'no hits recorded');
|
||||
return `${body.exact.entries} entries, ${body.exact.hits} hits, ${body.exact.tokensSaved} tokens saved`;
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log(`\n${C.B}guard and probing${C.x}`);
|
||||
|
||||
await check('GET /api/alerts', async () => {
|
||||
const body = await (await fetch(`${B}/api/alerts`)).json();
|
||||
assert(Array.isArray(body.alerts), 'no alerts array');
|
||||
assert(Array.isArray(body.forecasts), 'no forecasts');
|
||||
return `${body.alerts.length} alerts, ${body.forecasts.length} forecasts`;
|
||||
});
|
||||
|
||||
await check('POST /api/providers/:id/probe discovers models', async () => {
|
||||
const body = await (await post('/api/providers/alpha/probe', { validate: false })).json();
|
||||
const result = body.results[0];
|
||||
eq(result.auth, 'ok', 'auth');
|
||||
assert(result.discovered.length > 0, 'discovered nothing');
|
||||
return `${result.discovered.length} models discovered`;
|
||||
});
|
||||
|
||||
await check('content scanning flags a planted key', async () => {
|
||||
const res = await chat({ model: 'alpha/alpha-chat', messages: [{ role: 'user', content: 'deploy with sk-abcdefghijklmnopqrstuvwxyz12' }] });
|
||||
eq(res.status, 200, 'flag mode should not block');
|
||||
const log = store.logs(1)[0];
|
||||
return `logged, request still served (mode=${config.guard.scan.mode})`;
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log(`\n${C.B}dashboard API${C.x}`);
|
||||
|
||||
await check('GET /api/providers masks every key', async () => {
|
||||
const body = await (await fetch(`${B}/api/providers`)).json();
|
||||
const raw = JSON.stringify(body);
|
||||
assert(!raw.includes('ak-smoke-0001'), 'a raw provider key leaked into /api/providers');
|
||||
assert(!raw.includes('bk-smoke-0001'), 'a raw provider key leaked into /api/providers');
|
||||
const alphaCard = body.providers.find((p) => p.id === 'alpha');
|
||||
eq(alphaCard.keys.length, 2, 'key pool size');
|
||||
return `keys shown as ${alphaCard.keys[0].masked}`;
|
||||
});
|
||||
|
||||
await check('GET /api/stats', async () => {
|
||||
const body = await (await fetch(`${B}/api/stats`)).json();
|
||||
eq(body.providers.total, 2, 'provider count');
|
||||
return `${body.requests.ok} ok / ${body.requests.fail} failed`;
|
||||
});
|
||||
|
||||
await check('GET /api/logs', async () => {
|
||||
const body = await (await fetch(`${B}/api/logs?limit=10`)).json();
|
||||
assert(body.logs.length > 0, 'log is empty');
|
||||
return `${body.logs.length} entries`;
|
||||
});
|
||||
|
||||
await check('GET /api/events streams live', async () => {
|
||||
const controller = new AbortController();
|
||||
const res = await fetch(`${B}/api/events`, { signal: controller.signal, headers: { accept: 'text/event-stream' } });
|
||||
eq(res.status, 200, 'status');
|
||||
|
||||
const reader = res.body.getReader();
|
||||
const first = await reader.read();
|
||||
const text = new TextDecoder().decode(first.value);
|
||||
assert(text.includes('event: hello'), 'no hello frame');
|
||||
controller.abort();
|
||||
return 'SSE connected';
|
||||
});
|
||||
|
||||
await check('POST /api/providers/:id/enabled toggles', async () => {
|
||||
await post('/api/providers/alpha/enabled', { enabled: false });
|
||||
const res = await chat({ model: 'fast' });
|
||||
eq(res.headers.get('x-freeway-provider'), 'beta', 'should route around the disabled provider');
|
||||
await post('/api/providers/alpha/enabled', { enabled: true });
|
||||
return 'routed around, then restored';
|
||||
});
|
||||
|
||||
await check('POST /api/reload', async () => {
|
||||
const body = await (await post('/api/reload', {})).json();
|
||||
eq(body.ok, true, 'ok');
|
||||
return `${body.providers} providers reloaded`;
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log(`\n${C.B}shipped provider catalog${C.x}`);
|
||||
|
||||
await check('every providers/*.json parses', async () => {
|
||||
const real = new Registry({ dir: 'providers', env: {}, logger: silentLogger });
|
||||
const errors = real.issues().filter((i) => i.level === 'error');
|
||||
assert(errors.length === 0, errors.map((e) => `${e.file}: ${e.message}`).join('; '));
|
||||
return `${real.all().length} providers, ${real.allModels().length} models`;
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
server.close();
|
||||
context.close();
|
||||
cache.close();
|
||||
await alpha.close();
|
||||
await beta.close();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
|
||||
const colour = failed === 0 ? C.g : C.r;
|
||||
console.log(`\n${colour}${C.B}${failed === 0 ? 'PASS' : 'FAIL'}${C.x} — ${passed} passed, ${failed} failed\n`);
|
||||
if (failed > 0) {
|
||||
for (const f of failures) console.log(` ${C.r}${f.name}${C.x}: ${f.message}`);
|
||||
}
|
||||
process.exit(failed === 0 ? 0 : 1);
|
||||
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Check a provider file against the live API and print a verdict.
|
||||
*
|
||||
* node scripts/verify-provider.mjs groq
|
||||
* node scripts/verify-provider.mjs groq --validate # also send 1-token probes
|
||||
*
|
||||
* This is the feedback loop. A person — or an agent — adding a provider they
|
||||
* have never seen before should be able to run this, read the output, edit the
|
||||
* JSON, and run it again until it says PASS, without reading any TypeScript.
|
||||
*
|
||||
* Exit codes: 0 pass, 1 fail, 2 usage error.
|
||||
*/
|
||||
|
||||
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
|
||||
import { Registry } from '../packages/core/src/registry.ts';
|
||||
import { probeKey, limitDrift } from '../packages/core/src/probe.ts';
|
||||
|
||||
// The gateway reads .env at startup; this tool has to as well, or it reports
|
||||
// "no API key found" for a provider that is actually configured.
|
||||
loadDotEnv('.env');
|
||||
function loadDotEnv(path) {
|
||||
if (!existsSync(path)) return;
|
||||
for (const line of readFileSync(path, 'utf8').split('\n')) {
|
||||
const t = line.trim();
|
||||
if (!t || t.startsWith('#')) continue;
|
||||
const eq = t.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
const key = t.slice(0, eq).trim().replace(/^export\s+/, '');
|
||||
let value = t.slice(eq + 1).trim();
|
||||
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) value = value.slice(1, -1);
|
||||
if (process.env[key] === undefined) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const id = args.find((a) => !a.startsWith('--'));
|
||||
const validate = args.includes('--validate');
|
||||
const json = args.includes('--json');
|
||||
const apply = args.includes('--apply');
|
||||
|
||||
if (!id) {
|
||||
console.error(`
|
||||
Usage: node scripts/verify-provider.mjs <id> [--validate] [--json]
|
||||
|
||||
--validate send a 1-token request per model to see which the key can reach.
|
||||
Costs real quota. Without it, only discovery and auth are checked.
|
||||
--apply write the result back into providers/<id>.json. With --validate
|
||||
it keeps ONLY the models your key can actually reach, which is
|
||||
how you find out what a free tier really gives you.
|
||||
--json machine-readable output.
|
||||
`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const C = process.stdout.isTTY && !json
|
||||
? { r: '\x1b[31m', g: '\x1b[32m', y: '\x1b[33m', b: '\x1b[34m', d: '\x1b[2m', B: '\x1b[1m', x: '\x1b[0m' }
|
||||
: { r: '', g: '', y: '', b: '', d: '', B: '', x: '' };
|
||||
|
||||
const log = (...a) => { if (!json) console.log(...a); };
|
||||
const pass = (m) => log(` ${C.g}✓${C.x} ${m}`);
|
||||
const warn = (m) => log(` ${C.y}!${C.x} ${m}`);
|
||||
const fail = (m) => log(` ${C.r}✗${C.x} ${m}`);
|
||||
|
||||
const problems = [];
|
||||
const warnings = [];
|
||||
|
||||
// ---- 1. does the file parse against the schema? ---------------------------
|
||||
|
||||
const silent = { info() {}, warn() {}, error() {} };
|
||||
const registry = new Registry({ dir: 'providers', env: process.env, logger: silent });
|
||||
|
||||
log(`\n${C.B}Verifying provider "${id}"${C.x}\n`);
|
||||
log(`${C.B}1. file${C.x}`);
|
||||
|
||||
const fileIssues = registry.issues().filter((i) => i.providerId === id || i.file === `${id}.json`);
|
||||
for (const issue of fileIssues) {
|
||||
if (issue.level === 'error') { fail(issue.message); problems.push(issue.message); }
|
||||
else { warn(issue.message); warnings.push(issue.message); }
|
||||
}
|
||||
|
||||
const provider = registry.get(id);
|
||||
if (!provider) {
|
||||
fail(`providers/${id}.json did not load — fix the errors above`);
|
||||
if (fileIssues.length === 0) fail(`no such file: providers/${id}.json`);
|
||||
finish();
|
||||
}
|
||||
pass(`providers/${id}.json parses`);
|
||||
pass(`${provider.models.length} model${provider.models.length === 1 ? '' : 's'} declared, adapter=${provider.spec.adapter}`);
|
||||
|
||||
// ---- 2. environment --------------------------------------------------------
|
||||
|
||||
log(`\n${C.B}2. environment${C.x}`);
|
||||
if (!provider.configured) {
|
||||
fail(provider.configError);
|
||||
problems.push(provider.configError);
|
||||
finish();
|
||||
}
|
||||
pass(`baseUrl resolves to ${provider.baseUrl}`);
|
||||
pass(
|
||||
provider.spec.auth.type === 'none'
|
||||
? 'no credential required (auth.type: none)'
|
||||
: `${provider.keys.length} key${provider.keys.length === 1 ? '' : 's'} in the pool: ${provider.keys.map((k) => `${k.masked} (${k.source})`).join(', ')}`,
|
||||
);
|
||||
|
||||
// ---- 3. live probe ---------------------------------------------------------
|
||||
|
||||
log(`\n${C.B}3. live probe${C.x}${validate ? '' : ` ${C.d}(add --validate to test each model)${C.x}`}`);
|
||||
|
||||
const key = provider.keys[0] ?? { id: `${id}#none`, providerId: id, index: 0, value: '', masked: '—', source: 'none' };
|
||||
const result = await probeKey(provider, key, { validateModels: validate, maxModels: 80, concurrency: 2 });
|
||||
|
||||
const authOk = result.auth === 'ok';
|
||||
if (authOk) pass(`auth accepted (${result.ms}ms)`);
|
||||
else {
|
||||
const msg = `auth ${result.auth}${result.errors.length ? ` — ${result.errors[0]}` : ''}`;
|
||||
fail(msg);
|
||||
problems.push(msg);
|
||||
}
|
||||
|
||||
if (provider.spec.modelsEndpoint === null) {
|
||||
warn('discovery disabled (modelsEndpoint: null) — the declared model list is all there is');
|
||||
} else if (result.discovered.length > 0) {
|
||||
pass(`discovery found ${result.discovered.length} models via ${provider.spec.modelsEndpoint} → ${provider.spec.modelsPath}`);
|
||||
const declared = provider.models.map((m) => m.spec.id);
|
||||
const missing = result.discovered.filter((m) => !declared.includes(m));
|
||||
const stale = declared.filter((m) => !result.discovered.includes(m));
|
||||
|
||||
if (stale.length > 0) {
|
||||
warn(`declared but NOT offered by the API: ${stale.join(', ')}`);
|
||||
warnings.push(`stale model ids: ${stale.join(', ')}`);
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
log(` ${C.d}${missing.length} available ids are not in the file (see the suggestion below)${C.x}`);
|
||||
}
|
||||
} else if (result.errors.length > 0) {
|
||||
fail(result.errors.join('; '));
|
||||
problems.push(result.errors[0]);
|
||||
}
|
||||
|
||||
if (validate && result.models.length > 0) {
|
||||
const ok = result.models.filter((m) => m.status === 'ok');
|
||||
const bad = result.models.filter((m) => m.status !== 'ok' && m.status !== 'undiscovered');
|
||||
pass(`${ok.length}/${result.models.length} models answered`);
|
||||
for (const m of bad) {
|
||||
warn(`${m.id} → ${m.status}${m.error ? ` (${m.error.slice(0, 80)})` : ''}`);
|
||||
warnings.push(`${m.id}: ${m.status}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 4. observed limits ----------------------------------------------------
|
||||
|
||||
log(`\n${C.B}4. limits${C.x}`);
|
||||
const observed = result.observedLimits;
|
||||
const observedKeys = Object.keys(observed);
|
||||
|
||||
if (observedKeys.length === 0) {
|
||||
warn('the provider sent no rate-limit headers — declared limits cannot be checked against reality');
|
||||
} else {
|
||||
pass(`observed from response headers: ${observedKeys.map((k) => `${k}=${observed[k]}`).join(', ')}`);
|
||||
const drift = limitDrift(provider.spec.limits, observed);
|
||||
for (const d of drift) {
|
||||
if (d.declared === null) {
|
||||
log(` ${C.b}+${C.x} ${d.key}: file says null, provider reports ${d.observed} — consider recording it`);
|
||||
} else {
|
||||
warn(`${d.key}: file says ${d.declared}, provider reports ${d.observed}`);
|
||||
warnings.push(`${d.key} drift: ${d.declared} → ${d.observed}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (provider.spec.limitsSource === 'unverified' && observedKeys.length > 0) {
|
||||
log(` ${C.d}limitsSource is "unverified"; the observed values above could make it "observed"${C.x}`);
|
||||
}
|
||||
|
||||
// ---- 5. suggestion ---------------------------------------------------------
|
||||
|
||||
const suggestion = buildSuggestion();
|
||||
if (suggestion && !apply) {
|
||||
log(`\n${C.B}5. suggested edits${C.x}${C.d} (re-run with --apply to write these)${C.x}`);
|
||||
log(`${C.d}${JSON.stringify(suggestion, null, 2).split('\n').map((l) => ` ${l}`).join('\n')}${C.x}`);
|
||||
}
|
||||
|
||||
if (apply) applyToFile();
|
||||
|
||||
finish();
|
||||
|
||||
/**
|
||||
* Write what the live API reported back into the provider file.
|
||||
*
|
||||
* With --validate this is the answer to "which of these 55 models does my free
|
||||
* key actually get?": every id is probed and only the ones that answered are
|
||||
* kept. Existing aliases, caps and context windows are preserved — those encode
|
||||
* judgement the API cannot tell us.
|
||||
*/
|
||||
function applyToFile() {
|
||||
const path = `providers/${id}.json`;
|
||||
const file = JSON.parse(readFileSync(path, 'utf8'));
|
||||
const before = file.models.length;
|
||||
|
||||
const existing = new Map(file.models.map((m) => [m.id, m]));
|
||||
const reachable = validate
|
||||
? new Set(result.models.filter((m) => m.status === 'ok').map((m) => m.id))
|
||||
: new Set([...existing.keys(), ...result.discovered]);
|
||||
|
||||
// Keep a declared model the probe could not reach only when we never asked;
|
||||
// a validated miss is real evidence it does not work on this tier.
|
||||
const kept = [];
|
||||
const dropped = [];
|
||||
for (const [mid, model] of existing) {
|
||||
if (reachable.has(mid)) kept.push(model);
|
||||
else if (validate) dropped.push(mid);
|
||||
else kept.push(model);
|
||||
}
|
||||
|
||||
const added = [];
|
||||
const skipped = [];
|
||||
if (validate) {
|
||||
for (const m of result.models) {
|
||||
if (existing.has(m.id)) continue;
|
||||
|
||||
// Only a 200 to an actual chat completion proves a chat capability. A 400
|
||||
// ('exists') means the id is real but rejected a chat body — that is what
|
||||
// an OCR or text-to-speech model does, and tagging it "chat" would let
|
||||
// `auto` route a conversation into a transcription endpoint.
|
||||
if (m.status !== 'ok') {
|
||||
if (m.status === 'exists') skipped.push(m.id);
|
||||
continue;
|
||||
}
|
||||
const caps = capsFor(m.id);
|
||||
if (caps === null) { skipped.push(m.id); continue; }
|
||||
|
||||
kept.push({ id: m.id, label: m.id, alias: [], caps, context: null, priority: 50 });
|
||||
added.push(m.id);
|
||||
}
|
||||
}
|
||||
|
||||
file.models = kept;
|
||||
if (Object.keys(observed).length > 0) {
|
||||
file.limits = { ...file.limits, ...observed };
|
||||
file.limitsSource = 'observed';
|
||||
}
|
||||
file.verifiedOn = new Date().toISOString().slice(0, 10);
|
||||
|
||||
writeFileSync(path, `${JSON.stringify(file, null, 2)}\n`);
|
||||
|
||||
log(`\n${C.B}5. applied to ${path}${C.x}`);
|
||||
log(` ${C.g}✓${C.x} ${before} models -> ${kept.length}`);
|
||||
if (added.length > 0) log(` ${C.b}+${C.x} added ${added.length}: ${added.slice(0, 8).join(', ')}${added.length > 8 ? '…' : ''}`);
|
||||
if (dropped.length > 0) log(` ${C.y}-${C.x} removed ${dropped.length} your key cannot reach: ${dropped.join(', ')}`);
|
||||
if (skipped.length > 0) log(` ${C.d}skipped ${skipped.length} non-chat ids (ocr/audio/moderation/unproven): ${skipped.slice(0, 6).join(', ')}${skipped.length > 6 ? '…' : ''}${C.x}`);
|
||||
if (!validate) log(` ${C.d}run with --validate too, to keep only what your key can actually reach${C.x}`);
|
||||
log(` ${C.d}review aliases/caps/context by hand — the API cannot report those${C.x}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function buildSuggestion() {
|
||||
const out = {};
|
||||
const declared = provider.models.map((m) => m.spec.id);
|
||||
const missing = result.discovered.filter((m) => !declared.includes(m));
|
||||
|
||||
if (missing.length > 0) {
|
||||
out.models = missing.slice(0, 12).map((mid) => ({
|
||||
id: mid,
|
||||
// Capability guesses from the id are a starting point, not an answer.
|
||||
caps: /embed/i.test(mid) ? ['embed'] : ['chat'],
|
||||
context: null,
|
||||
alias: [],
|
||||
}));
|
||||
}
|
||||
if (Object.keys(observed).length > 0) {
|
||||
out.limits = { ...provider.spec.limits, ...observed };
|
||||
out.limitsSource = 'observed';
|
||||
out.verifiedOn = new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
return Object.keys(out).length > 0 ? out : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Guess capabilities from an id. Returns null for model families that are not
|
||||
* chat or embeddings at all, so they are left out of the file entirely rather
|
||||
* than mislabelled.
|
||||
*/
|
||||
function capsFor(modelId) {
|
||||
const m = modelId.toLowerCase();
|
||||
if (/(^|[-/])(embed|embedding)/.test(m)) return ['embed'];
|
||||
// Audio, vision-to-text, and safety endpoints share the chat URL but do not
|
||||
// take a chat body.
|
||||
if (/ocr|tts|transcribe|voxtral|whisper|moderation|rerank|guard|shield/.test(m)) return null;
|
||||
|
||||
const caps = ['chat'];
|
||||
if (/code|codestral|devstral|fim/.test(m)) caps.push('code');
|
||||
if (/magistral|reason|think|-r1|qwq/.test(m)) caps.push('reasoning');
|
||||
return caps;
|
||||
}
|
||||
|
||||
function finish() {
|
||||
const verdict = problems.length === 0 ? 'PASS' : 'FAIL';
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify({ id, verdict, problems, warnings, probe: typeof result === 'undefined' ? null : result }, null, 2));
|
||||
} else {
|
||||
const colour = problems.length === 0 ? C.g : C.r;
|
||||
log(`\n${colour}${C.B}${verdict}${C.x} — ${problems.length} problem${problems.length === 1 ? '' : 's'}, ${warnings.length} warning${warnings.length === 1 ? '' : 's'}\n`);
|
||||
}
|
||||
process.exit(problems.length === 0 ? 0 : 1);
|
||||
}
|
||||
Reference in new issue
Block a user