Aryan SinghandClaude Opus 4.8
b2ae3d6ced
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016DvdRZkyCy9NJpHYDdPqmS
91 lines
3.2 KiB
Python
91 lines
3.2 KiB
Python
"""Phase-1 GATE: does the local model reliably emit tool calls over MULTIPLE rounds?
|
|
|
|
This is the single load-bearing risk of the whole design. A single tool call often works;
|
|
the failure mode is multi-turn (the model stops calling tools, or emits malformed calls).
|
|
Run this with LM Studio up and a model loaded:
|
|
|
|
lms server start && lms load gpt-oss-20b
|
|
uv run python scripts/smoke_toolcalling.py
|
|
|
|
PASS = the model chained >=3 real tool calls and produced a final answer.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pathlib
|
|
import sys
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
|
|
|
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
|
|
|
|
from llm.client import build_llm, list_loaded_models, resolve_model_id
|
|
from tools.market_data import get_quote
|
|
from tools.python_exec import run_python
|
|
from tools.web import web_search
|
|
|
|
TOOLS = [run_python, web_search, get_quote]
|
|
TOOLS_BY_NAME = {t.name: t for t in TOOLS}
|
|
|
|
SYSTEM = (
|
|
"You are a test agent. Use a tool for EVERY factual or numeric step — never compute "
|
|
"in your head. Call tools one step at a time, then give a final summary."
|
|
)
|
|
TASK = (
|
|
"Do these steps, each via a tool call, then summarize the results:\n"
|
|
"1) Get the last price of AAPL.\n"
|
|
"2) With run_python, compute a 7% stop-loss price (price * 0.93).\n"
|
|
"3) With run_python, compute a 15% take-profit price (price * 1.15).\n"
|
|
"4) web_search for one recent Apple (AAPL) news headline.\n"
|
|
"Report the stop price, take price, and the headline."
|
|
)
|
|
MAX_ROUNDS = 10
|
|
|
|
|
|
def main() -> int:
|
|
loaded = list_loaded_models()
|
|
if not loaded:
|
|
print("FAIL: LM Studio not reachable at /v1/models.")
|
|
print("Start it: lms server start then lms load <model>")
|
|
return 2
|
|
print("Loaded models :", loaded)
|
|
print("Resolved model:", resolve_model_id())
|
|
print("-" * 60)
|
|
|
|
llm = build_llm(TOOLS)
|
|
messages: list = [SystemMessage(content=SYSTEM), HumanMessage(content=TASK)]
|
|
tool_calls_made = 0
|
|
rounds = 0
|
|
|
|
while rounds < MAX_ROUNDS:
|
|
rounds += 1
|
|
ai: AIMessage = llm.invoke(messages)
|
|
messages.append(ai)
|
|
if not ai.tool_calls:
|
|
print(f"\n[round {rounds}] FINAL:\n{ai.content}\n")
|
|
break
|
|
for tc in ai.tool_calls:
|
|
name, args = tc["name"], tc["args"]
|
|
print(f"[round {rounds}] call {name}({args})")
|
|
tool = TOOLS_BY_NAME.get(name)
|
|
if tool is None:
|
|
result = f"ERROR: unknown tool {name}"
|
|
else:
|
|
try:
|
|
result = tool.invoke(args)
|
|
except Exception as e:
|
|
result = f"ERROR: {type(e).__name__}: {e}"
|
|
tool_calls_made += 1
|
|
print(f" -> {str(result)[:180]}")
|
|
messages.append(ToolMessage(content=str(result), tool_call_id=tc["id"]))
|
|
|
|
print("=" * 60)
|
|
print(f"rounds={rounds} tool_calls_executed={tool_calls_made}")
|
|
ok = tool_calls_made >= 3
|
|
print("RESULT:", "PASS — multi-turn tool calling works" if ok else "FAIL — tool calling unreliable")
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|