feat: pin research subagents to DeepSeek Flash
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { visibleWidth } from "@earendil-works/pi-tui";
|
||||
import { buildTreeRows, parseAgentModel, parseAgentTranscript, renderTranscriptLines, shortAgentName } from "../src/agent-chat-panel.js";
|
||||
import type { AgentRecord } from "../src/types.js";
|
||||
|
||||
const SESSION_HEADER = [
|
||||
'{"type":"session","version":3,"id":"s-1","timestamp":"2026-07-31T00:00:00.000Z","cwd":"/proj"}',
|
||||
'{"type":"model_change","id":"m-1","parentId":null,"timestamp":"2026-07-31T00:00:00.000Z","provider":"deepseek","modelId":"deepseek-v4-flash"}',
|
||||
].join("\n");
|
||||
|
||||
const USER_MSG = '{"type":"message","id":"u-1","parentId":null,"timestamp":"2026-07-31T00:00:01.000Z","message":{"role":"user","content":[{"type":"text","text":"Research the market"}]}}';
|
||||
|
||||
const ASSISTANT_MSG = '{"type":"message","id":"a-1","parentId":"u-1","timestamp":"2026-07-31T00:00:02.000Z","message":{"role":"assistant","content":[{"type":"thinking","thinking":"I should spawn a child."},{"type":"text","text":"I will delegate."},{"type":"toolCall","id":"call_1","name":"spawn_agent","arguments":"{\\"name\\":\\"critic\\"}"}],"model":"deepseek-v4-flash"}}';
|
||||
|
||||
const TOOL_RESULT_MSG = '{"type":"message","id":"t-1","parentId":"a-1","timestamp":"2026-07-31T00:00:03.000Z","message":{"role":"toolResult","toolCallId":"call_1","toolName":"spawn_agent","content":[{"type":"text","text":"{\\"status\\":\\"completed\\"}"}]}}';
|
||||
|
||||
const MALFORMED = "{not json";
|
||||
|
||||
function writeSession(dir: string, lines: string[]): string {
|
||||
const file = join(dir, "agent.jsonl");
|
||||
writeFileSync(file, lines.join("\n") + "\n", "utf8");
|
||||
return file;
|
||||
}
|
||||
|
||||
describe("parseAgentTranscript", () => {
|
||||
it("extracts user, assistant, thinking, tool calls and tool results in order", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-transcript-"));
|
||||
try {
|
||||
const file = writeSession(dir, [SESSION_HEADER, USER_MSG, ASSISTANT_MSG, TOOL_RESULT_MSG, MALFORMED].flatMap((line) => [line]));
|
||||
const entries = parseAgentTranscript(file);
|
||||
expect(entries.map((entry) => entry.role)).toEqual(["user", "assistant", "thinking", "tool", "tool"]);
|
||||
expect(entries[0]).toMatchObject({ role: "user", label: "user", text: "Research the market" });
|
||||
expect(entries[1]).toMatchObject({ role: "assistant", label: "assistant", text: "I will delegate." });
|
||||
expect(entries[2]).toMatchObject({ role: "thinking", label: "thinking" });
|
||||
expect(entries[2]?.text).toContain("spawn a child");
|
||||
expect(entries[3]).toMatchObject({ role: "tool", label: "→ spawn_agent" });
|
||||
expect(entries[3]?.text).toContain('"name": "critic"');
|
||||
expect(entries[4]).toMatchObject({ role: "tool", label: "← spawn_agent" });
|
||||
expect(entries[4]?.text).toContain('"status":"completed"');
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores non-message entries and malformed lines", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-transcript-"));
|
||||
try {
|
||||
const file = writeSession(dir, [SESSION_HEADER, MALFORMED]);
|
||||
expect(parseAgentTranscript(file)).toEqual([]);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("reads the actual provider/model from a model-change entry", () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "hm-transcript-"));
|
||||
try {
|
||||
const file = writeSession(dir, [SESSION_HEADER, ASSISTANT_MSG]);
|
||||
expect(parseAgentModel(file)).toBe("deepseek/deepseek-v4-flash");
|
||||
expect(parseAgentModel(undefined)).toBeUndefined();
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to the task when the file is missing or absent", () => {
|
||||
expect(parseAgentTranscript(undefined)).toEqual([]);
|
||||
expect(parseAgentTranscript(undefined, "Do the thing")).toEqual([{ role: "assistant", label: "task", text: "Do the thing" }]);
|
||||
expect(parseAgentTranscript("/no/such/file.jsonl", "Fallback")).toEqual([{ role: "assistant", label: "task", text: "Fallback" }]);
|
||||
});
|
||||
});
|
||||
|
||||
const base: Omit<AgentRecord, "id" | "status"> = {
|
||||
runId: "run-test", parentId: "root", children: [], lineage: [], depth: 0,
|
||||
task: "t", taskFingerprint: "t", expectedOutput: "", completionCriteria: "", specPath: "",
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
const agent = (id: string, status: AgentRecord["status"], parentId: string | null, depth: number, startedMin: number, finishedMin?: number): AgentRecord => {
|
||||
const record: AgentRecord = { ...base, id, parentId, depth, status, startedAt: new Date(Date.now() - startedMin * 60_000).toISOString() };
|
||||
if (finishedMin !== undefined) record.finishedAt = new Date(Date.now() - finishedMin * 60_000).toISOString();
|
||||
return record;
|
||||
};
|
||||
|
||||
describe("buildTreeRows", () => {
|
||||
it("flattens depth-first with depth and short names", () => {
|
||||
const root = agent("supervisor-91e05755", "created", null, 0, 0);
|
||||
const lead = agent("arithmetic-lead-80d7cee9", "running", root.id, 1, 3);
|
||||
const calc = agent("independent-calculator-39d08b41", "completed", lead.id, 2, 5, 2);
|
||||
root.children = [lead.id]; lead.children = [calc.id];
|
||||
const rows = buildTreeRows([root, lead, calc], root.id, Date.now());
|
||||
expect(rows.map((row) => row.id)).toEqual([root.id, lead.id, calc.id]);
|
||||
expect(rows.map((row) => row.depth)).toEqual([0, 1, 2]);
|
||||
expect(rows.map((row) => row.name)).toEqual(["supervisor", "arithmetic-lead", "independent-calculator"]);
|
||||
expect(rows[1]?.elapsed).toMatch(/^\d{2}:\d{2}$/);
|
||||
expect(rows[0]?.elapsed).toBe("");
|
||||
expect(rows[1]?.status).toBe("running");
|
||||
expect(rows[1]?.hasSession).toBe(false);
|
||||
});
|
||||
|
||||
it("computes elapsed from start to finish for completed agents", () => {
|
||||
const lead = agent("arithmetic-lead-80d7cee9", "completed", "supervisor-91e05755", 1, 10, 4);
|
||||
const root = agent("supervisor-91e05755", "created", null, 0, 0);
|
||||
root.children = [lead.id];
|
||||
const rows = buildTreeRows([root, lead], root.id, Date.now());
|
||||
expect(rows[1]?.elapsed).toBe("06:00");
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderTranscriptLines", () => {
|
||||
it("adds role separators and wraps content to the width", () => {
|
||||
const entries = [
|
||||
{ role: "user" as const, label: "user", text: "short" },
|
||||
{ role: "assistant" as const, label: "assistant", text: "word ".repeat(40) },
|
||||
];
|
||||
const width = 30;
|
||||
const lines = renderTranscriptLines(entries, width);
|
||||
expect(lines[0]).toMatchObject({ kind: "separator", role: "user" });
|
||||
expect(lines[0]?.text.startsWith("── user")).toBe(true);
|
||||
expect(lines[1]).toMatchObject({ kind: "content", role: "user", text: "short" });
|
||||
for (const line of lines) expect(visibleWidth(line.text)).toBeLessThanOrEqual(width);
|
||||
expect(lines.filter((line) => line.role === "assistant" && line.kind === "content").length).toBeGreaterThan(2);
|
||||
});
|
||||
|
||||
it("stays width-safe with long labels", () => {
|
||||
const lines = renderTranscriptLines([{ role: "tool" as const, label: "← some_really_long_tool_name", text: "x" }], 20);
|
||||
expect(visibleWidth(lines[0]!.text)).toBeLessThanOrEqual(20);
|
||||
});
|
||||
});
|
||||
|
||||
describe("shortAgentName", () => {
|
||||
it("strips the trailing 8-hex id", () => {
|
||||
expect(shortAgentName("arithmetic-lead-80d7cee9")).toBe("arithmetic-lead");
|
||||
expect(shortAgentName("plain")).toBe("plain");
|
||||
});
|
||||
});
|
||||
@@ -22,5 +22,5 @@ describe("AgentTree", () => {
|
||||
it("does not count cancelled or failed children toward the child limit", async () => { const { tree } = setup({ ...DEFAULT_CONFIG, max_children_per_agent: 2 }); const first = await tree.spawn(request(tree.rootId, "Alpha", "First concrete assignment in a limited tree")); await tree.cancel(first.id); const second = await tree.spawn(request(tree.rootId, "Beta", "Second concrete assignment in a limited tree")); await tree.cancel(second.id); await expect(tree.spawn(request(tree.rootId, "Gamma", "Third concrete assignment in a limited tree"))).resolves.toBeTruthy(); });
|
||||
it("cancel does not clobber an already recorded result", async () => { const { tree } = setup(); const child = await tree.spawn(request(tree.rootId, "Done", "Complete a concrete assignment and return")); const result = await tree.start(child.id); expect(result.status).toBe("completed"); await tree.cancel(child.id); expect(tree.inspect(child.id).status).toBe("cancelled"); expect(tree.inspect(child.id).result?.status).toBe("completed"); });
|
||||
it("fails agents that exceed the configured timeout", async () => { const { tree } = setup({ ...DEFAULT_CONFIG, agent_timeout_seconds: 0.05 }, 200); const child = await tree.spawn(request(tree.rootId, "Slow", "Run a deliberately slow concrete assignment")); const result = await tree.start(child.id); expect(result.status).toBe("failed"); expect(result.summary).toMatch(/timed out/); expect(tree.inspect(child.id).error).toMatch(/timed out/); });
|
||||
it("caps how many agent sessions stream at the same time", async () => { const { tree } = setup({ ...DEFAULT_CONFIG, agent_concurrency: 1 }, 40); const a = await tree.spawn(request(tree.rootId, "First", "Sequential streaming assignment number one")); const b = await tree.spawn(request(tree.rootId, "Second", "Sequential streaming assignment number two")); const [ra, rb] = await Promise.all([tree.start(a.id), tree.start(b.id)]); expect(ra.status).toBe("completed"); expect(rb.status).toBe("completed"); const startedA = Date.parse(tree.inspect(a.id).startedAt!); const startedB = Date.parse(tree.inspect(b.id).startedAt!); const finishedA = Date.parse(tree.inspect(a.id).finishedAt!); expect(startedB).toBeGreaterThanOrEqual(finishedA - 5); });
|
||||
it("caps how many agent sessions stream at the same time", async () => { const { tree } = setup({ ...DEFAULT_CONFIG, agent_concurrency: 1 }, 40); const a = await tree.spawn(request(tree.rootId, "First", "Sequential streaming assignment number one")); const b = await tree.spawn(request(tree.rootId, "Second", "Sequential streaming assignment number two")); const [ra, rb] = await Promise.all([tree.start(a.id), tree.start(b.id)]); expect(ra.status).toBe("completed"); expect(rb.status).toBe("completed"); const startedB = Date.parse(tree.inspect(b.id).startedAt!); const finishedA = Date.parse(tree.inspect(a.id).finishedAt!); expect(startedB).toBeGreaterThanOrEqual(finishedA - 5); });
|
||||
});
|
||||
|
||||
@@ -2,5 +2,5 @@ import { describe, expect, it } from "vitest";
|
||||
import extension from "../src/index.js";
|
||||
|
||||
describe("Pi extension smoke", () => {
|
||||
it("loads as an extension factory and registers all commands", () => { const commands: string[] = []; const events: string[] = []; const renderers: string[] = []; const fakePi = { on: (name: string) => events.push(name), registerCommand: (name: string) => commands.push(name), registerMessageRenderer: (name: string) => renderers.push(name) } as any; extension(fakePi); expect(commands).toEqual(expect.arrayContaining(["team", "research", "research-status", "research-pause", "research-resume", "research-stop", "findings", "hypotheses"])); expect(events).toEqual(expect.arrayContaining(["session_start", "session_shutdown"])); expect(renderers).toContain("hypothesis-machine-agent-update"); });
|
||||
it("loads as an extension factory and registers all commands", () => { const commands: string[] = []; const events: string[] = []; const renderers: string[] = []; const fakePi = { on: (name: string) => events.push(name), registerCommand: (name: string) => commands.push(name), registerMessageRenderer: (name: string) => renderers.push(name) } as any; extension(fakePi); expect(commands).toEqual(expect.arrayContaining(["team", "agent", "agent-open", "research", "research-status", "research-pause", "research-resume", "research-stop", "findings", "hypotheses"])); expect(events).toEqual(expect.arrayContaining(["session_start", "session_shutdown"])); expect(renderers).toContain("hypothesis-machine-agent-update"); });
|
||||
});
|
||||
|
||||
@@ -18,8 +18,9 @@ describe("PiAgentRuntimeFactory", () => {
|
||||
const model: Model<any> = { id: "fake-model", name: "Fake model", api: "openai-completions", provider: "hm-fake", baseUrl: "http://invalid.test", reasoning: false, input: ["text"], cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 32_000, maxTokens: 2_000 };
|
||||
const runtime = await ModelRuntime.create({ authPath: resolve(cwd, "auth.json"), modelsPath: null });
|
||||
runtime.registerNativeProvider({ id: "hm-fake", name: "HM fake provider", auth: { apiKey: { name: "fake", resolve: async () => ({ auth: { apiKey: "not-a-real-secret" }, source: "test" }) } }, getModels: () => [model], stream: () => { throw new Error("simple stream expected"); }, streamSimple: () => { const stream = createAssistantMessageEventStream(); const message: AssistantMessage = { role: "assistant", content: [{ type: "text", text: "fake child result" }], api: model.api, provider: model.provider, model: model.id, usage: { input: 1, output: 3, cacheRead: 0, cacheWrite: 0, totalTokens: 4, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, stopReason: "stop", timestamp: Date.now() }; queueMicrotask(() => stream.end(message)); return stream; } });
|
||||
const stateDir = resolve(cwd, ".hypothesis-machine"); const store = new RunStore(stateDir); const memory = new ResearchMemory(stateDir); const web = new WebGateway(DEFAULT_CONFIG, memory); const experiments = new ExperimentRunner(stateDir, DEFAULT_CONFIG.experiment);
|
||||
const factory = new PiAgentRuntimeFactory({ cwd, config: DEFAULT_CONFIG, store, memory, web, experiments, modelRuntime: runtime, model, thinkingLevel: "off" }); const tree = new AgentTree(store, factory, DEFAULT_CONFIG, { goal: "Test the official child runtime" }); factory.attachTree(tree);
|
||||
const config = { ...DEFAULT_CONFIG, subagent_model: "hm-fake/fake-model" };
|
||||
const stateDir = resolve(cwd, ".hypothesis-machine"); const store = new RunStore(stateDir); const memory = new ResearchMemory(stateDir); const web = new WebGateway(config, memory); const experiments = new ExperimentRunner(stateDir, config.experiment);
|
||||
const factory = new PiAgentRuntimeFactory({ cwd, config, store, memory, web, experiments, modelRuntime: runtime, thinkingLevel: "off" }); const tree = new AgentTree(store, factory, config, { goal: "Test the official child runtime" }); factory.attachTree(tree);
|
||||
const child = await tree.spawn({ parentId: tree.rootId, name: "Runtime Child", role: "Runtime verifier", task: "Return the deterministic fake model response", expectedOutput: "Text result", completionCriteria: "A response is persisted", tools: [], background: false });
|
||||
const result = await tree.start(child.id); expect(result.summary).toBe("fake child result"); expect(tree.inspect(child.id).sessionFile).toMatch(/\.jsonl$/); expect(tree.inspect(child.id).status).toBe("completed"); memory.close();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { mkdtempSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { AgentTree } from "../src/agent-tree.js";
|
||||
import { readAgentSpec } from "../src/agent-spec.js";
|
||||
import { DEFAULT_CONFIG, DEFAULT_SUBAGENT_MODEL } from "../src/config.js";
|
||||
import { RunStore } from "../src/run-store.js";
|
||||
import { createResearchTools } from "../src/tools/index.js";
|
||||
import { FakeRuntimeFactory } from "./helpers.js";
|
||||
|
||||
describe("subagent model selection", () => {
|
||||
it("pins spawned agents to DeepSeek V4 Flash instead of the caller model", async () => {
|
||||
const cwd = mkdtempSync(resolve(tmpdir(), "hm-subagent-model-"));
|
||||
const store = new RunStore(cwd);
|
||||
const tree = new AgentTree(store, new FakeRuntimeFactory(), DEFAULT_CONFIG, { goal: "Test subagent model routing" });
|
||||
const spawn = createResearchTools({ tree, parentId: tree.rootId, memory: {} as any, web: {} as any, experiments: {} as any, cwd }).find((tool) => tool.name === "spawn_agent")!;
|
||||
|
||||
await spawn.execute("spawn-1", {
|
||||
name: "Direct API verifier", role: "routing verifier", task: "Verify that subagents use the configured direct model", expected_output: "Agent specification", completion_criteria: "Model is pinned", background: false,
|
||||
}, undefined, undefined, { model: { provider: "openai", id: "gpt-5.6-terra" }, thinkingLevel: "high" } as any);
|
||||
|
||||
const child = tree.list().find((agent) => agent.id !== tree.rootId)!;
|
||||
expect(readAgentSpec(child.specPath).model).toBe(DEFAULT_SUBAGENT_MODEL);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user