Emil_Shanaty/hypothesis-machine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: pin DNS resolution against SSRF rebinding; enforce reviewer independence; bound caches and logs

Browse Source 
- web.ts: outbound downloads resolve DNS once, validate the address, and
  connect to the validated IP directly, closing the DNS-rebinding TOCTOU
  window; malformed SearXNG result URLs are skipped instead of failing the
  whole search; web cache is bounded (evicts oldest past 500 entries).
- experiment.ts: reviewer must not be a descendant of the experiment author
  (tool-level lineage check on top of the author-id check); stdout/stderr
  buffers capped at 512 KB.
- Confirmed Pi's extension tool registry is per-extension-instance, so
  reload re-registration is safe; no change needed there.
- Adds a lineage-independence test (39 tests passing, tsc clean).
This commit is contained in:
Emil
2026-07-31 23:03:49 +03:00
parent 18b8a3247d
commit 4039f7c8f3
5 changed files with 83 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+11
View File
@@ -15,6 +15,17 @@
- Full-text search now prefix-matches tokens (tolerates Russian/English
inflections without stemming) and safely handles punctuation and FTS5
metacharacters instead of throwing or returning false negatives.
- Hardened the web gateway against SSRF TOCTOU: outbound downloads now resolve
DNS once, validate the address, and connect to the validated IP directly
(pinned resolution), so DNS rebinding cannot redirect the connection to a
private target; malformed search results no longer break the whole query;
the web cache is bounded (oldest entries evicted past 500 files).
- Experiment review is now truly independent: the reviewer must not be a
descendant of the experiment author (in addition to not being the author).
- Experiment stdout/stderr are capped at 512 KB so runaway container output
cannot exhaust memory or disk.
- Verified that Pi's extension tool registry is per-extension-instance, so
reload re-registration is safe (no fix required).
## 0.1.1 — 2026-07-31
src/tools/experiment.ts
+10 -1
View File
@@ -15,8 +15,10 @@ export function dockerArguments(config: HypothesisMachineConfig["experiment"], d
return ["run", "--rm", ...(containerName ? ["--name", containerName] : []), "--network", "none", "--read-only", "--cpus", String(config.cpus), "--memory", `${config.memory_mb}m`, "--pids-limit", "128", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--tmpfs", "/tmp:rw,noexec,nosuid,size=64m", "-v", `${directory}:/workspace:ro`, "-v", `${resolve(directory, "artifacts")}:/workspace/artifacts:rw`, "-w", "/workspace", image, "sh", "-lc", command];
}
const MAX_LOG_BYTES = 512 * 1024;
async function runProcess(command: string, args: string[], timeoutMs: number, onTimeout?: () => void): Promise<{ code: number; stdout: string; stderr: string; timeout: boolean }> {
return new Promise((resolvePromise, reject) => { const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], env: { PATH: process.env.PATH ?? "/usr/bin:/bin" } }); let stdout = "", stderr = "", timeout = false; child.stdout.on("data", (chunk) => stdout += String(chunk)); child.stderr.on("data", (chunk) => stderr += String(chunk)); const timer = setTimeout(() => { timeout = true; onTimeout?.(); child.kill("SIGKILL"); }, timeoutMs); child.on("error", reject); child.on("close", (code) => { clearTimeout(timer); resolvePromise({ code: code ?? 1, stdout, stderr, timeout }); }); });
return new Promise((resolvePromise, reject) => { const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"], env: { PATH: process.env.PATH ?? "/usr/bin:/bin" } }); let stdout = "", stderr = "", timeout = false; child.stdout.on("data", (chunk) => { stdout = (stdout + String(chunk)).slice(-MAX_LOG_BYTES); }); child.stderr.on("data", (chunk) => { stderr = (stderr + String(chunk)).slice(-MAX_LOG_BYTES); }); const timer = setTimeout(() => { timeout = true; onTimeout?.(); child.kill("SIGKILL"); }, timeoutMs); child.on("error", reject); child.on("close", (code) => { clearTimeout(timer); resolvePromise({ code: code ?? 1, stdout, stderr, timeout }); }); });
}
export class ExperimentRunner {
@@ -40,6 +42,13 @@ export class ExperimentRunner {
return { id, status: result.timeout ? "timeout" : result.code === 0 ? "completed" : "failed", exitCode: result.code, directory, planHash };
}
authorOf(experimentId: string): string {
if (!/^exp-[a-f0-9]{8}$/.test(experimentId)) throw new Error("Invalid experiment id");
const manifestPath = resolve(this.stateDir, "experiments", experimentId, "experiment-manifest.json");
if (!existsSync(manifestPath)) throw new Error(`Unknown experiment: ${experimentId}`);
return (JSON.parse(readFileSync(manifestPath, "utf8")) as { createdBy: string }).createdBy;
}
review(input: ExperimentReview): { experimentId: string; status: HypothesisStatus; reviewPath: string } {
if (!/^exp-[a-f0-9]{8}$/.test(input.experimentId)) throw new Error("Invalid experiment id");
const directory = resolve(this.stateDir, "experiments", input.experimentId); const manifestPath = resolve(directory, "experiment-manifest.json");
src/tools/index.ts
+1 -1
View File
@@ -33,6 +33,6 @@ export function createResearchTools(deps: ToolDeps): ToolDefinition[] {
const webBrowse = defineTool({ name: "web_browse", label: "Interactive browser fallback", description: "Use the configured local Browser Use adapter for an interactive public page.", parameters: Type.Object({ url: Type.String(), task: Type.String() }), async execute(_id, params) { return text(await deps.web.browse(params.url, params.task)); } });
const download = defineTool({ name: "download_source", label: "Download source", description: "Download, hash, validate, deduplicate, and preserve a public source.", parameters: Type.Object({ url: Type.String() }), async execute(_id, params) { return text(await deps.web.downloadSource(params.url)); } });
const experiment = defineTool({ name: "run_experiment", label: "Run isolated experiment", description: "Run a frozen test plan and generated source in a networkless, resource-limited Docker container. Never falls back to host execution.", executionMode: "sequential" as const, parameters: Type.Object({ plan: Type.Object({ hypothesis: Type.String(), data: Type.String(), baseline: Type.String(), split: Type.String(), metrics: Type.Array(Type.String()), successCriterion: Type.String(), refutationCriterion: Type.String(), confounders: Type.Array(Type.String()), resourceLimits: Type.String() }), command: Type.String(), source_files: Type.Record(Type.String(), Type.String()), data_manifest: Type.Optional(Type.Unknown()), image: Type.Optional(Type.String()) }), async execute(_id, params) { return text(await deps.experiments.run({ plan: params.plan, command: params.command, sourceFiles: params.source_files, createdBy: deps.parentId, ...(params.data_manifest !== undefined ? { dataManifest: params.data_manifest } : {}), ...(params.image ? { image: params.image } : {}) })); } });
const reviewExperiment = defineTool({ name: "review_experiment", label: "Review experiment independently", description: "Record an independent verdict for a frozen experiment. The experiment author cannot review their own work.", executionMode: "sequential" as const, parameters: Type.Object({ experiment_id: Type.String(), verdict: StringEnum(["supported", "partially_supported", "inconclusive", "contradicted", "invalid_experiment", "requires_external_validation"] as const), summary: Type.String(), limitations: Type.String() }), async execute(_id, params) { return text(deps.experiments.review({ experimentId: params.experiment_id, reviewerId: deps.parentId, verdict: params.verdict, summary: params.summary, limitations: params.limitations })); } });
const reviewExperiment = defineTool({ name: "review_experiment", label: "Review experiment independently", description: "Record an independent verdict for a frozen experiment. The experiment author cannot review their own work.", executionMode: "sequential" as const, parameters: Type.Object({ experiment_id: Type.String(), verdict: StringEnum(["supported", "partially_supported", "inconclusive", "contradicted", "invalid_experiment", "requires_external_validation"] as const), summary: Type.String(), limitations: Type.String() }), async execute(_id, params) { const authorId = deps.experiments.authorOf(params.experiment_id); const reviewer = deps.tree.inspect(deps.parentId); if (reviewer.lineage.includes(authorId)) throw new Error("Independent review must be performed by an agent that is not a descendant of the experiment author"); return text(deps.experiments.review({ experimentId: params.experiment_id, reviewerId: deps.parentId, verdict: params.verdict, summary: params.summary, limitations: params.limitations })); } });
return [spawn, control, memorySearch, publish, readArtifact, webSearch, webRead, webCrawl, webBrowse, download, experiment, reviewExperiment];
}
src/tools/web.ts
+53 -12
View File
@@ -1,6 +1,8 @@
import { createHash } from "node:crypto";
import { promises as dns } from "node:dns";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, unlinkSync, writeFileSync } from "node:fs";
import { request as httpRequest, type IncomingHttpHeaders, type RequestOptions } from "node:http";
import { request as httpsRequest } from "node:https";
import { isIP } from "node:net";
import { resolve } from "node:path";
import type { HypothesisMachineConfig } from "../config.js";
@@ -35,20 +37,49 @@ export async function assertPublicUrl(raw: string): Promise<string> {
return normalized;
}
async function limitedFetch(url: string, init: RequestInit, timeoutMs: number, maxBytes: number): Promise<{ response: Response; bytes: Buffer; finalUrl: string }> {
async function limitedFetch(url: string, init: { headers?: Record<string, string> }, timeoutMs: number, maxBytes: number): Promise<{ response: { status: number; headers: IncomingHttpHeaders }; bytes: Buffer; finalUrl: string }> {
let current = url;
for (let redirects = 0; redirects <= 5; redirects++) {
current = await assertPublicUrl(current); const response = await fetch(current, { ...init, redirect: "manual", signal: AbortSignal.timeout(timeoutMs) });
if (response.status >= 300 && response.status < 400) { const location = response.headers.get("location"); if (!location) throw new Error("Redirect missing Location header"); current = new URL(location, current).toString(); continue; }
if (!response.ok) throw new Error(`HTTP ${response.status} from ${new URL(current).origin}`);
const declared = Number(response.headers.get("content-length") ?? 0); if (declared > maxBytes) throw new Error(`Content exceeds ${maxBytes} byte limit`);
const reader = response.body?.getReader(); const chunks: Uint8Array[] = []; let total = 0;
if (reader) while (true) { const { done, value } = await reader.read(); if (done) break; total += value.byteLength; if (total > maxBytes) { await reader.cancel(); throw new Error(`Content exceeds ${maxBytes} byte limit`); } chunks.push(value); }
return { response, bytes: Buffer.concat(chunks), finalUrl: current };
current = await assertPublicUrl(current);
const { status, headers, body } = await pinnedGet(new URL(current), init.headers ?? {}, timeoutMs, maxBytes);
if (status >= 300 && status < 400) { const location = headers.location; if (!location) throw new Error("Redirect missing Location header"); current = new URL(location, current).toString(); continue; }
if (status < 200 || status >= 300) throw new Error(`HTTP ${status} from ${new URL(current).origin}`);
return { response: { status, headers }, bytes: body, finalUrl: current };
}
throw new Error("Too many redirects");
}
/** Resolve and validate a hostname once, then connect to the validated address so DNS rebinding cannot redirect the connection to a private target. */
async function pinnedGet(url: URL, headers: Record<string, string>, timeoutMs: number, maxBytes: number): Promise<{ status: number; headers: IncomingHttpHeaders; body: Buffer }> {
const addresses = await dns.lookup(url.hostname, { all: true });
const publicAddresses = addresses.map((entry) => entry.address).filter((address) => !isPrivateAddress(address));
if (!publicAddresses.length) throw new Error("Blocked private, local, or reserved network target");
const address = publicAddresses[0]!;
const isIpv6 = address.includes(":");
const defaultPort = url.protocol === "https:" ? 443 : 80;
const port = url.port ? Number(url.port) : defaultPort;
const hostHeader = url.port && url.port !== String(defaultPort) ? `${url.hostname}:${url.port}` : url.hostname;
const requestOptions: RequestOptions & { servername?: string } = {
protocol: url.protocol, hostname: isIpv6 ? address.replace(/^\[|\]$/g, "") : address, port,
path: `${url.pathname}${url.search}`, method: "GET", headers: { ...headers, Host: hostHeader },
...(url.protocol === "https:" ? { servername: url.hostname } : {}),
};
const request = url.protocol === "https:" ? httpsRequest : httpRequest;
return await new Promise<{ status: number; headers: IncomingHttpHeaders; body: Buffer }>((resolvePromise, reject) => {
const req = request(requestOptions, (response) => {
const status = response.statusCode ?? 0;
const declared = Number(response.headers["content-length"] ?? 0); if (declared > maxBytes) { response.destroy(); reject(new Error(`Content exceeds ${maxBytes} byte limit`)); return; }
const chunks: Buffer[] = []; let total = 0; let failed = false;
response.on("data", (chunk: Buffer) => { if (failed) return; total += chunk.length; if (total > maxBytes) { failed = true; response.destroy(); reject(new Error(`Content exceeds ${maxBytes} byte limit`)); return; } chunks.push(chunk); });
response.on("end", () => { if (!failed) resolvePromise({ status, headers: response.headers, body: Buffer.concat(chunks) }); });
response.on("error", (error) => { if (!failed) { failed = true; reject(error); } });
});
req.setTimeout(timeoutMs, () => req.destroy(new Error(`Request timed out after ${timeoutMs}ms`)));
req.on("error", (error) => reject(error));
req.end();
});
}
export interface WebDocument { url: string; title?: string; content: string; mime: string; sourceId: string; sha256: string; retrievedAt: string; untrusted: true; backend: string }
export class WebGateway {
@@ -66,7 +97,9 @@ export class WebGateway {
const url = new URL("/search", this.config.searxng_url); url.searchParams.set("q", query); url.searchParams.set("format", "json");
const response = await fetch(url, { signal: AbortSignal.timeout(this.config.web_timeout_ms) }); if (!response.ok) throw new Error(`SearXNG HTTP ${response.status}; run docker compose -f infra/compose.yaml up -d`);
const body = await response.json() as { results?: Array<{ title?: string; url?: string; content?: string }> };
return (body.results ?? []).filter((item): item is { title?: string; url: string; content?: string } => Boolean(item.url)).slice(0, Math.max(1, Math.min(50, limit))).map((item) => ({ title: item.title ?? item.url, url: normalizeUrl(item.url), snippet: item.content ?? "" }));
const results: Array<{ title: string; url: string; snippet: string }> = [];
for (const item of (body.results ?? [])) { if (!item.url) continue; try { results.push({ title: item.title ?? item.url, url: normalizeUrl(item.url), snippet: item.content ?? "" }); } catch { /* skip malformed result URLs */ } if (results.length >= Math.max(1, Math.min(50, limit))) break; }
return results;
}
async read(rawUrl: string, refresh = false): Promise<WebDocument> {
@@ -75,7 +108,7 @@ export class WebGateway {
if (!response.ok) throw new Error(`Firecrawl HTTP ${response.status}; verify the self-hosted service`);
const raw = await response.json() as any; const data = raw.data ?? raw; const content = String(data.markdown ?? data.content ?? ""); if (Buffer.byteLength(content) > this.config.max_download_bytes) throw new Error("Firecrawl response exceeds size limit");
const bytes = Buffer.from(content); const saved = this.memory.saveSource(url, bytes, "text/markdown"); const document: WebDocument = { url, title: data.metadata?.title, content, mime: "text/markdown", sourceId: saved.id, sha256: saved.hash, retrievedAt: new Date().toISOString(), untrusted: true, backend: "firecrawl" };
writeFileSync(cache, `${JSON.stringify(document, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); return document;
writeFileSync(cache, `${JSON.stringify(document, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); this.evictCache(500); return document;
}
async crawl(rawUrl: string, limit = 20): Promise<unknown> {
@@ -91,6 +124,14 @@ export class WebGateway {
async downloadSource(rawUrl: string): Promise<{ id: string; path: string; hash: string }> {
const url = await assertPublicUrl(rawUrl); const { response, bytes, finalUrl } = await limitedFetch(url, { headers: { "user-agent": "HypothesisMachine/0.1" } }, this.config.web_timeout_ms, this.config.max_download_bytes);
const mime = (response.headers.get("content-type") ?? "application/octet-stream").split(";")[0]!; if (!ALLOWED_MIME.test(mime)) throw new Error(`Blocked MIME type: ${mime}`); return this.memory.saveSource(finalUrl, bytes, mime);
const rawContentType = response.headers["content-type"]; const mime = (Array.isArray(rawContentType) ? rawContentType[0] : rawContentType ?? "application/octet-stream").split(";")[0]!; if (!ALLOWED_MIME.test(mime)) throw new Error(`Blocked MIME type: ${mime}`); return this.memory.saveSource(finalUrl, bytes, mime);
}
/** Bound the disk used by the web cache: evict oldest entries once the cap is exceeded. */
private evictCache(cap: number): void {
if (!existsSync(this.cacheDir)) return;
const files = readdirSync(this.cacheDir, { withFileTypes: true }).filter((entry) => entry.isFile() && entry.name.endsWith(".json")).map((entry) => ({ name: entry.name, path: resolve(this.cacheDir, entry.name), mtime: statSync(resolve(this.cacheDir, entry.name)).mtimeMs }));
if (files.length <= cap) return;
files.sort((a, b) => a.mtime - b.mtime);
for (const file of files.slice(0, files.length - cap)) { try { unlinkSync(file.path); } catch { /* best effort */ } }
}
}
tests/experiment.test.ts
+8
View File
@@ -2,11 +2,19 @@ import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { resolve } from "node:path";
import { describe, expect, it } from "vitest";
import { AgentTree } from "../src/agent-tree.js";
import { DEFAULT_CONFIG } from "../src/config.js";
import { ResearchMemory } from "../src/research-memory.js";
import { RunStore } from "../src/run-store.js";
import { createResearchTools } from "../src/tools/index.js";
import { WebGateway } from "../src/tools/web.js";
import { ExperimentRunner, dockerArguments, validateTestPlan } from "../src/tools/experiment.js";
import { FakeRuntimeFactory } from "./helpers.js";
const plan = { hypothesis: "A improves B", data: "fixed.csv", baseline: "mean", split: "predefined", metrics: ["rmse"], successCriterion: "rmse < 1", refutationCriterion: "rmse >= 1", confounders: ["leakage"], resourceLimits: "1 CPU" };
describe("ExperimentRunner", () => {
it("requires precommitted test criteria", () => { expect(() => validateTestPlan(plan)).not.toThrow(); expect(() => validateTestPlan({ ...plan, baseline: "" })).toThrow(/baseline/); });
it("constructs networkless resource-limited Docker arguments", () => { const args = dockerArguments({ image: "python", cpus: 1.5, memory_mb: 512, timeout_seconds: 30 }, "/tmp/exp", "python", "python source/test.py"); expect(args).toEqual(expect.arrayContaining(["--network", "none", "--read-only", "--cpus", "1.5", "--memory", "512m", "--cap-drop", "ALL"])); expect(args.join(" ")).not.toMatch(/\.pi|HOME|API_KEY/); });
it("requires a different agent for independent review", () => { const dir = mkdtempSync(resolve(tmpdir(), "hm-review-")); const expDir = resolve(dir, "experiments", "exp-aabbccdd"); mkdirSync(expDir, { recursive: true }); writeFileSync(resolve(expDir, "experiment-manifest.json"), JSON.stringify({ createdBy: "implementer", planHash: "abc", hypothesisStatus: "testing" })); const runner = new ExperimentRunner(dir, { image: "none", cpus: 1, memory_mb: 128, timeout_seconds: 1 }); expect(() => runner.review({ experimentId: "exp-aabbccdd", reviewerId: "implementer", verdict: "supported", summary: "Looks good", limitations: "Small sample" })).toThrow(/other than/); expect(runner.review({ experimentId: "exp-aabbccdd", reviewerId: "reviewer", verdict: "inconclusive", summary: "Metric is unstable", limitations: "Small sample" }).status).toBe("inconclusive"); });
it("rejects reviewers that descend from the experiment author", async () => { const dir = mkdtempSync(resolve(tmpdir(), "hm-review-lineage-")); const store = new RunStore(dir); const memory = new ResearchMemory(dir); const web = new WebGateway({ ...DEFAULT_CONFIG, state_dir: dir }, memory); const experiments = new ExperimentRunner(dir, { image: "none", cpus: 1, memory_mb: 128, timeout_seconds: 1 }); const tree = new AgentTree(store, new FakeRuntimeFactory(), DEFAULT_CONFIG, { goal: "Test experiment review independence" }); const author = await tree.spawn({ parentId: tree.rootId, name: "Author", role: "experiment author", task: "Run a concrete experiment about review independence", expectedOutput: "Experiment", completionCriteria: "Experiment executed", background: false }); const descendant = await tree.spawn({ parentId: author.id, name: "Reviewer", role: "reviewer", task: "Review the concrete experiment about review independence", expectedOutput: "Verdict", completionCriteria: "Verdict recorded", background: false }); const expDir = resolve(dir, "experiments", "exp-aabbccdd"); mkdirSync(expDir, { recursive: true }); writeFileSync(resolve(expDir, "experiment-manifest.json"), JSON.stringify({ createdBy: author.id, planHash: "abc", hypothesisStatus: "testing" })); const reviewTool = (parentId: string) => createResearchTools({ tree, parentId, memory, web, experiments, cwd: dir }).find((tool) => tool.name === "review_experiment")!; await expect(reviewTool(descendant.id).execute("1", { experiment_id: "exp-aabbccdd", verdict: "supported", summary: "ok", limitations: "small sample" }, undefined, undefined, {} as any)).rejects.toThrow(/not a descendant/); await expect(reviewTool(tree.rootId).execute("2", { experiment_id: "exp-aabbccdd", verdict: "inconclusive", summary: "unstable", limitations: "small sample" }, undefined, undefined, {} as any)).resolves.toBeTruthy(); memory.close(); });
});