Emil
4039f7c8f3
- web.ts: outbound downloads resolve DNS once, validate the address, and connect to the validated IP directly, closing the DNS-rebinding TOCTOU window; malformed SearXNG result URLs are skipped instead of failing the whole search; web cache is bounded (evicts oldest past 500 entries). - experiment.ts: reviewer must not be a descendant of the experiment author (tool-level lineage check on top of the author-id check); stdout/stderr buffers capped at 512 KB. - Confirmed Pi's extension tool registry is per-extension-instance, so reload re-registration is safe; no change needed there. - Adds a lineage-independence test (39 tests passing, tsc clean).
21 lines
4.2 KiB
TypeScript
21 lines
4.2 KiB
TypeScript
import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { resolve } from "node:path";
|
|
import { describe, expect, it } from "vitest";
|
|
import { AgentTree } from "../src/agent-tree.js";
|
|
import { DEFAULT_CONFIG } from "../src/config.js";
|
|
import { ResearchMemory } from "../src/research-memory.js";
|
|
import { RunStore } from "../src/run-store.js";
|
|
import { createResearchTools } from "../src/tools/index.js";
|
|
import { WebGateway } from "../src/tools/web.js";
|
|
import { ExperimentRunner, dockerArguments, validateTestPlan } from "../src/tools/experiment.js";
|
|
import { FakeRuntimeFactory } from "./helpers.js";
|
|
|
|
const plan = { hypothesis: "A improves B", data: "fixed.csv", baseline: "mean", split: "predefined", metrics: ["rmse"], successCriterion: "rmse < 1", refutationCriterion: "rmse >= 1", confounders: ["leakage"], resourceLimits: "1 CPU" };
|
|
describe("ExperimentRunner", () => {
|
|
it("requires precommitted test criteria", () => { expect(() => validateTestPlan(plan)).not.toThrow(); expect(() => validateTestPlan({ ...plan, baseline: "" })).toThrow(/baseline/); });
|
|
it("constructs networkless resource-limited Docker arguments", () => { const args = dockerArguments({ image: "python", cpus: 1.5, memory_mb: 512, timeout_seconds: 30 }, "/tmp/exp", "python", "python source/test.py"); expect(args).toEqual(expect.arrayContaining(["--network", "none", "--read-only", "--cpus", "1.5", "--memory", "512m", "--cap-drop", "ALL"])); expect(args.join(" ")).not.toMatch(/\.pi|HOME|API_KEY/); });
|
|
it("requires a different agent for independent review", () => { const dir = mkdtempSync(resolve(tmpdir(), "hm-review-")); const expDir = resolve(dir, "experiments", "exp-aabbccdd"); mkdirSync(expDir, { recursive: true }); writeFileSync(resolve(expDir, "experiment-manifest.json"), JSON.stringify({ createdBy: "implementer", planHash: "abc", hypothesisStatus: "testing" })); const runner = new ExperimentRunner(dir, { image: "none", cpus: 1, memory_mb: 128, timeout_seconds: 1 }); expect(() => runner.review({ experimentId: "exp-aabbccdd", reviewerId: "implementer", verdict: "supported", summary: "Looks good", limitations: "Small sample" })).toThrow(/other than/); expect(runner.review({ experimentId: "exp-aabbccdd", reviewerId: "reviewer", verdict: "inconclusive", summary: "Metric is unstable", limitations: "Small sample" }).status).toBe("inconclusive"); });
|
|
it("rejects reviewers that descend from the experiment author", async () => { const dir = mkdtempSync(resolve(tmpdir(), "hm-review-lineage-")); const store = new RunStore(dir); const memory = new ResearchMemory(dir); const web = new WebGateway({ ...DEFAULT_CONFIG, state_dir: dir }, memory); const experiments = new ExperimentRunner(dir, { image: "none", cpus: 1, memory_mb: 128, timeout_seconds: 1 }); const tree = new AgentTree(store, new FakeRuntimeFactory(), DEFAULT_CONFIG, { goal: "Test experiment review independence" }); const author = await tree.spawn({ parentId: tree.rootId, name: "Author", role: "experiment author", task: "Run a concrete experiment about review independence", expectedOutput: "Experiment", completionCriteria: "Experiment executed", background: false }); const descendant = await tree.spawn({ parentId: author.id, name: "Reviewer", role: "reviewer", task: "Review the concrete experiment about review independence", expectedOutput: "Verdict", completionCriteria: "Verdict recorded", background: false }); const expDir = resolve(dir, "experiments", "exp-aabbccdd"); mkdirSync(expDir, { recursive: true }); writeFileSync(resolve(expDir, "experiment-manifest.json"), JSON.stringify({ createdBy: author.id, planHash: "abc", hypothesisStatus: "testing" })); const reviewTool = (parentId: string) => createResearchTools({ tree, parentId, memory, web, experiments, cwd: dir }).find((tool) => tool.name === "review_experiment")!; await expect(reviewTool(descendant.id).execute("1", { experiment_id: "exp-aabbccdd", verdict: "supported", summary: "ok", limitations: "small sample" }, undefined, undefined, {} as any)).rejects.toThrow(/not a descendant/); await expect(reviewTool(tree.rootId).execute("2", { experiment_id: "exp-aabbccdd", verdict: "inconclusive", summary: "unstable", limitations: "small sample" }, undefined, undefined, {} as any)).resolves.toBeTruthy(); memory.close(); });
|
|
});
|