361 lines
14 KiB
Python
Executable File
361 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
export_public.py — SANITIZED public snapshot exporter for the trading dashboard.
|
|
|
|
Reads the local paper-trading state (SQLite + nightly committee JSON) and emits a
|
|
single, DECOUPLED, read-only file (`trading-public.json`) that the public Next.js
|
|
dashboard consumes. NOTHING sensitive leaves this machine:
|
|
|
|
STRIPPED — absolute $ equity / cash / buying-power, Alpaca account IDs,
|
|
Alpaca order IDs, per-share dollar prices, market values.
|
|
EMITTED — per-persona PERCENT returns, BASE-100 normalized equity curves,
|
|
thesis cards, committee verdicts, trade markers (no order IDs),
|
|
and positions as (symbol + side + RELATIVE weight) only.
|
|
|
|
The equity source (`snapshots`) is sparse (~6 rows/persona over ~10 days), so the
|
|
curves are labelled coarse/sample and step-interpolated by the dashboard — no
|
|
intraday points are fabricated here.
|
|
|
|
Stdlib only (sqlite3 + json). Run:
|
|
|
|
python3 scripts/export_public.py \
|
|
--db state/trading.db \
|
|
--out trading-public.json
|
|
|
|
A final self-check scans the emitted JSON for dollar figures / ID patterns and
|
|
aborts if any are found, so a sanitization regression can never ship silently.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
from collections import defaultdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Persona presentation metadata. `account`/`agent` keys in the DB are persona
|
|
# slugs (nova / warren_lite / atlas / maverick) — NOT Alpaca account IDs.
|
|
# --------------------------------------------------------------------------- #
|
|
PERSONA_ORDER = ["nova", "warren_lite", "atlas", "maverick"]
|
|
PERSONA_META = {
|
|
"nova": {"label": "Nova", "style": "momentum"},
|
|
"warren_lite": {"label": "Warren-lite", "style": "value"},
|
|
"atlas": {"label": "Atlas", "style": "macro"},
|
|
"maverick": {"label": "Maverick", "style": "high-beta"},
|
|
}
|
|
|
|
# Patterns that must NEVER appear in the emitted file.
|
|
_UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
|
|
|
|
|
def _round(x, n):
|
|
"""Round, coercing to float and tolerating None."""
|
|
try:
|
|
return round(float(x), n)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _short(text, limit=180):
|
|
"""Trim a rationale to a word boundary with an ellipsis.
|
|
|
|
Also strips '$' so no dollar glyph reaches the public file (rationales
|
|
occasionally reference strikes/prices like "$60 strike" — descriptive, but
|
|
the sanitized feed carries zero dollar signs by policy)."""
|
|
if not text:
|
|
return ""
|
|
t = " ".join(str(text).replace("$", "").split())
|
|
if len(t) <= limit:
|
|
return t
|
|
cut = t[:limit].rsplit(" ", 1)[0].rstrip(",.;:")
|
|
return cut + "…"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Loaders
|
|
# --------------------------------------------------------------------------- #
|
|
def load_snapshots(con):
|
|
"""{account: [(ts, equity), ...]} ordered by ts. Equity is used only to
|
|
derive PERCENT return + base-100 curve; the raw $ figure never leaves here."""
|
|
rows = con.execute(
|
|
"SELECT account, ts, equity FROM snapshots "
|
|
"WHERE equity IS NOT NULL ORDER BY account, ts"
|
|
).fetchall()
|
|
by_acct = defaultdict(list)
|
|
for account, ts, equity in rows:
|
|
by_acct[account].append((ts, float(equity)))
|
|
return by_acct
|
|
|
|
|
|
def build_curve(series):
|
|
"""Base-100 normalized curve (one point per date, last snapshot of the day).
|
|
|
|
Returns (points, return_pct, n_snapshots). Base = first snapshot's equity."""
|
|
if not series:
|
|
return [], None, 0
|
|
base = series[0][1]
|
|
if base == 0:
|
|
return [], None, len(series)
|
|
|
|
per_date = {} # date -> equity of last snapshot that day (insertion keeps last)
|
|
for ts, equity in series:
|
|
per_date[ts[:10]] = equity
|
|
|
|
points = [{"t": d, "v": _round(eq / base * 100.0, 2)} for d, eq in per_date.items()]
|
|
last_equity = series[-1][1]
|
|
return_pct = _round((last_equity / base - 1.0) * 100.0, 1)
|
|
return points, return_pct, len(series)
|
|
|
|
|
|
def load_positions(con):
|
|
"""{account: [{symbol, side, weight}, ...]} — RELATIVE weights only.
|
|
|
|
weight = |market_value| / sum(|market_value|) per account; the absolute $
|
|
market_value is used only for the ratio and is never emitted."""
|
|
rows = con.execute(
|
|
"SELECT account, symbol, qty, market_value FROM positions "
|
|
"WHERE market_value IS NOT NULL"
|
|
).fetchall()
|
|
raw = defaultdict(list)
|
|
for account, symbol, qty, mv in rows:
|
|
raw[account].append((symbol, float(qty or 0), abs(float(mv))))
|
|
|
|
out = {}
|
|
for account, items in raw.items():
|
|
total = sum(mv for _, _, mv in items) or 1.0
|
|
packed = [
|
|
{
|
|
"symbol": sym,
|
|
"side": "short" if qty < 0 else "long",
|
|
"weight": _round(mv / total, 4),
|
|
}
|
|
for sym, qty, mv in items
|
|
]
|
|
packed.sort(key=lambda p: p["weight"], reverse=True)
|
|
out[account] = packed
|
|
return out
|
|
|
|
|
|
def load_theses(con):
|
|
"""Deduped thesis cards: one row per (date, persona, symbol), highest
|
|
conviction wins. Returns a list plus a lookup of the chosen direction."""
|
|
rows = con.execute(
|
|
"SELECT trade_date, agent, ticker, direction, conviction, thesis "
|
|
"FROM theses ORDER BY trade_date, agent, ticker"
|
|
).fetchall()
|
|
|
|
best = {} # (date, persona, symbol) -> row dict
|
|
for trade_date, agent, ticker, direction, conviction, thesis in rows:
|
|
key = (trade_date, agent, ticker)
|
|
conv = _round(conviction, 2) or 0.0
|
|
if key not in best or conv > best[key]["conviction"]:
|
|
best[key] = {
|
|
"date": trade_date,
|
|
"persona": agent,
|
|
"symbol": ticker,
|
|
"direction": (direction or "long").lower(),
|
|
"conviction": conv,
|
|
"rationaleShort": _short(thesis),
|
|
}
|
|
|
|
theses = sorted(best.values(), key=lambda t: (t["date"], t["persona"], t["symbol"]))
|
|
direction_by_key = {(t["date"], t["persona"], t["symbol"]): t["direction"] for t in theses}
|
|
return theses, direction_by_key
|
|
|
|
|
|
def load_committee(output_dir):
|
|
"""Committee verdicts from state/output/<date>/_committee.json files.
|
|
|
|
Keys look like 'nova|ASML'. Returns (list, verdict_lookup)."""
|
|
committee = []
|
|
verdict_by_key = {}
|
|
if not output_dir.is_dir():
|
|
return committee, verdict_by_key
|
|
|
|
for date_dir in sorted(output_dir.iterdir()):
|
|
cfile = date_dir / "_committee.json"
|
|
if not cfile.is_file():
|
|
continue
|
|
date = date_dir.name
|
|
try:
|
|
data = json.loads(cfile.read_text())
|
|
except (json.JSONDecodeError, OSError):
|
|
continue
|
|
for key, val in data.items():
|
|
if key.startswith("_") or "|" not in key: # skip _memo etc.
|
|
continue
|
|
persona, symbol = key.split("|", 1)
|
|
verdict = str(val.get("verdict", "")).lower()
|
|
if verdict not in ("approve", "reduce", "veto"):
|
|
continue
|
|
committee.append({
|
|
"date": date,
|
|
"persona": persona,
|
|
"symbol": symbol,
|
|
"verdict": verdict,
|
|
"note": _short(val.get("reason", ""), 200),
|
|
})
|
|
verdict_by_key[(date, persona, symbol)] = verdict
|
|
|
|
committee.sort(key=lambda c: (c["date"], c["persona"], c["symbol"]))
|
|
return committee, verdict_by_key
|
|
|
|
|
|
def build_trades(direction_by_key, verdict_by_key):
|
|
"""Trade markers derived from theses + committee (NOT order status/IDs).
|
|
|
|
A thesis becomes a trade unless the committee vetoed it. side = buy|sell,
|
|
mapped from the thesis direction (long->buy, short->sell)."""
|
|
trades = []
|
|
for (date, persona, symbol), direction in direction_by_key.items():
|
|
if verdict_by_key.get((date, persona, symbol)) == "veto":
|
|
continue
|
|
trades.append({
|
|
"date": date,
|
|
"persona": persona,
|
|
"symbol": symbol,
|
|
"side": "sell" if direction == "short" else "buy",
|
|
})
|
|
trades.sort(key=lambda t: (t["date"], t["persona"], t["symbol"]))
|
|
return trades
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Assembly + sanitization gate
|
|
# --------------------------------------------------------------------------- #
|
|
def build_payload(db_path, output_dir):
|
|
con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
|
try:
|
|
snaps = load_snapshots(con)
|
|
positions = load_positions(con)
|
|
theses, direction_by_key = load_theses(con)
|
|
finally:
|
|
con.close()
|
|
|
|
committee, verdict_by_key = load_committee(output_dir)
|
|
trades = build_trades(direction_by_key, verdict_by_key)
|
|
|
|
# as_of = latest snapshot date seen anywhere.
|
|
all_dates = [ts[:10] for series in snaps.values() for ts, _ in series]
|
|
as_of = max(all_dates) if all_dates else None
|
|
|
|
personas = []
|
|
snapshot_total = 0
|
|
for pid in PERSONA_ORDER:
|
|
curve, ret, n = build_curve(snaps.get(pid, []))
|
|
snapshot_total += n
|
|
meta = PERSONA_META.get(pid, {"label": pid, "style": "unknown"})
|
|
personas.append({
|
|
"id": pid,
|
|
"label": meta["label"],
|
|
"style": meta["style"],
|
|
"returnPct": ret,
|
|
"snapshotCount": n,
|
|
"equityCurve": curve,
|
|
"positions": positions.get(pid, []),
|
|
})
|
|
|
|
return {
|
|
"generatedAt": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
"asOf": as_of,
|
|
"personas": personas,
|
|
"theses": theses,
|
|
"committee": committee,
|
|
"trades": trades,
|
|
"meta": {
|
|
"note": "Paper trading (Alpaca PAPER). Sanitized, decoupled snapshot — no live account data.",
|
|
"curveNote": "Equity curves are base-100 normalized and COARSE/SAMPLE: "
|
|
"~6 sparse snapshots per persona over ~10 days, step-interpolated. Not intraday.",
|
|
"snapshotCount": snapshot_total,
|
|
"personaCount": len(personas),
|
|
"thesisCount": len(theses),
|
|
"committeeCount": len(committee),
|
|
"tradeCount": len(trades),
|
|
},
|
|
}
|
|
|
|
|
|
def assert_sanitized(payload, db_path):
|
|
"""Fail loudly if any $ figure, account/order UUID, or raw account-level
|
|
dollar value leaked into the emitted JSON."""
|
|
blob = json.dumps(payload)
|
|
problems = []
|
|
|
|
if "$" in blob:
|
|
problems.append("literal '$' present")
|
|
m = _UUID_RE.search(blob)
|
|
if m:
|
|
problems.append(f"UUID-like id present: {m.group(0)[:8]}…")
|
|
|
|
# Cross-check against the actual account-level dollar figures in the DB.
|
|
con = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)
|
|
try:
|
|
dollars = set()
|
|
for (v,) in con.execute("SELECT equity FROM snapshots UNION "
|
|
"SELECT cash FROM snapshots UNION "
|
|
"SELECT buying_power FROM snapshots UNION "
|
|
"SELECT market_value FROM positions"):
|
|
if v is None:
|
|
continue
|
|
fv = float(v)
|
|
if abs(fv) >= 50: # account/position dollar scale; avoids weight/conviction collisions
|
|
dollars.add(repr(round(fv, 2)))
|
|
dollars.add(repr(fv))
|
|
finally:
|
|
con.close()
|
|
|
|
for d in dollars:
|
|
needle = d.rstrip("0").rstrip(".") if "." in d else d
|
|
if needle and needle in blob:
|
|
problems.append(f"raw dollar figure present: {needle}")
|
|
break
|
|
|
|
if problems:
|
|
raise SystemExit("SANITIZATION FAILED: " + "; ".join(problems))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description="Export a sanitized public trading snapshot.")
|
|
parser.add_argument("--db", default="state/trading.db", help="Path to trading.db (default: state/trading.db)")
|
|
parser.add_argument("--out", default="trading-public.json", help="Output JSON path (default: ./trading-public.json)")
|
|
parser.add_argument("--output-dir", default=None,
|
|
help="Nightly output dir with <date>/_committee.json (default: <db_dir>/output)")
|
|
args = parser.parse_args(argv)
|
|
|
|
db_path = Path(args.db).expanduser().resolve()
|
|
if not db_path.is_file():
|
|
raise SystemExit(f"database not found: {db_path}")
|
|
|
|
output_dir = Path(args.output_dir).expanduser().resolve() if args.output_dir \
|
|
else db_path.parent / "output"
|
|
|
|
payload = build_payload(db_path, output_dir)
|
|
assert_sanitized(payload, db_path)
|
|
|
|
out_path = Path(args.out).expanduser().resolve()
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
out_path.write_text(json.dumps(payload, indent=2) + "\n")
|
|
|
|
# Summary — non-sensitive lines only (percentages + counts, never $).
|
|
print(f"wrote {out_path}")
|
|
print(f"asOf {payload['asOf']} snapshots {payload['meta']['snapshotCount']}")
|
|
for p in payload["personas"]:
|
|
ret = p["returnPct"]
|
|
rstr = f"{ret:+.1f}%" if ret is not None else "n/a"
|
|
print(f" {p['id']:<12} {rstr:>7} curve {len(p['equityCurve'])}pt positions {len(p['positions'])}")
|
|
print(f"theses {payload['meta']['thesisCount']} "
|
|
f"committee {payload['meta']['committeeCount']} "
|
|
f"trades {payload['meta']['tradeCount']}")
|
|
print("sanitization: PASS (no $, no account/order IDs)")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|