313 lines
10 KiB
TypeScript
313 lines
10 KiB
TypeScript
import { promises as fs } from "node:fs";
|
|
import path from "node:path";
|
|
import { spawn, type ChildProcess } from "node:child_process";
|
|
import { fileURLToPath } from "node:url";
|
|
import { randomUUID, createHash } from "node:crypto";
|
|
import { unzipSync } from "fflate";
|
|
import { gameArchive } from "../engine/archive.ts";
|
|
import { clone, type Project } from "../engine/schema.ts";
|
|
import { normalizeOptions } from "../native/options.mjs";
|
|
import { doctor } from "../native/build.mjs";
|
|
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
export type BuildJob = {
|
|
id: string;
|
|
status: "queued" | "building" | "succeeded" | "failed" | "cancelled";
|
|
revision: number;
|
|
projectId: string;
|
|
createdAt: string;
|
|
finishedAt?: string;
|
|
options: ReturnType<typeof normalizeOptions>;
|
|
logs: string[];
|
|
error?: string;
|
|
artifacts?: {
|
|
name: string;
|
|
bytes: number;
|
|
sha256: string;
|
|
url: string;
|
|
path: string;
|
|
}[];
|
|
};
|
|
export class BuildManager {
|
|
private jobs = new Map<string, BuildJob>();
|
|
private queue: { id: string; project: Project }[] = [];
|
|
private child: ChildProcess | null = null;
|
|
private running: string | null = null;
|
|
private stopped = false;
|
|
private idle: Promise<void> = Promise.resolve();
|
|
private saved: Promise<void> = Promise.resolve();
|
|
readonly directory: string;
|
|
constructor(
|
|
projectDir: string,
|
|
private read: (uri: string) => Promise<Uint8Array>,
|
|
) {
|
|
this.directory = path.join(projectDir, "builds");
|
|
}
|
|
async init() {
|
|
await fs.mkdir(this.directory, { recursive: true });
|
|
for (const n of await fs.readdir(this.directory))
|
|
if (/^[a-f0-9-]{36}$/.test(n)) {
|
|
try {
|
|
const j = JSON.parse(
|
|
await fs.readFile(path.join(this.directory, n, "job.json"), "utf8"),
|
|
) as BuildJob;
|
|
if (j.id !== n) continue;
|
|
if (["queued", "building"].includes(j.status)) {
|
|
j.status = "failed";
|
|
j.error = "Build interrupted by server restart";
|
|
j.finishedAt = new Date().toISOString();
|
|
await this.persist(j);
|
|
}
|
|
this.jobs.set(n, j);
|
|
} catch {
|
|
/* An unfinished metadata write must not prevent opening the project. */
|
|
}
|
|
}
|
|
}
|
|
capabilities() {
|
|
return doctor();
|
|
}
|
|
list() {
|
|
return [...this.jobs.values()]
|
|
.sort((a, b) => b.createdAt.localeCompare(a.createdAt))
|
|
.slice(0, 30)
|
|
.map((j) => ({ ...j, logs: j.logs.slice(-80) }));
|
|
}
|
|
get(id: string) {
|
|
const j = this.jobs.get(id);
|
|
if (!j) throw Error("Build not found");
|
|
return structuredClone(j);
|
|
}
|
|
private persist(job: BuildJob) {
|
|
const data = JSON.stringify(job, null, 2),
|
|
file = path.join(this.directory, job.id, "job.json");
|
|
this.saved = this.saved
|
|
.catch(() => {})
|
|
.then(async () => {
|
|
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
await fs.writeFile(file + ".tmp", data, { mode: 0o600 });
|
|
await fs.rename(file + ".tmp", file);
|
|
});
|
|
return this.saved;
|
|
}
|
|
async start(project: Project, raw: any, expectedRevision: number) {
|
|
if (this.stopped) throw Error("Build service closed");
|
|
if (expectedRevision !== project.revision)
|
|
throw Error("REVISION_CONFLICT: reread project before building");
|
|
if (this.queue.length >= 3)
|
|
throw Error("Build queue full (maximum 3 waiting jobs)");
|
|
const options = normalizeOptions(raw),
|
|
cap = await this.capabilities();
|
|
if (!cap.targets[options.target as keyof typeof cap.targets].ready)
|
|
throw Error(
|
|
"BUILD_TOOLS_MISSING: " +
|
|
cap.targets[options.target as keyof typeof cap.targets].missing.join(
|
|
"; ",
|
|
),
|
|
);
|
|
if (
|
|
options.target === "android" &&
|
|
options.mode === "release" &&
|
|
!cap.targets.android.releaseSigningConfigured
|
|
)
|
|
throw Error(
|
|
"RELEASE_SIGNING_MISSING: configure signing in the local server environment",
|
|
);
|
|
if (expectedRevision !== project.revision) throw Error("REVISION_CONFLICT");
|
|
if (this.stopped) throw Error("Build service closed");
|
|
if (this.queue.length >= 3) throw Error("Build queue full");
|
|
const job: BuildJob = {
|
|
id: randomUUID(),
|
|
status: "queued",
|
|
revision: project.revision,
|
|
projectId: project.id,
|
|
createdAt: new Date().toISOString(),
|
|
options,
|
|
logs: ["Snapshot revision " + project.revision],
|
|
};
|
|
this.jobs.set(job.id, job);
|
|
this.queue.push({ id: job.id, project: clone(project) });
|
|
await this.persist(job);
|
|
this.pump();
|
|
return this.get(job.id);
|
|
}
|
|
private pump() {
|
|
if (this.running || this.stopped) return;
|
|
const next = this.queue.shift();
|
|
if (!next) return;
|
|
this.running = next.id;
|
|
this.idle = this.execute(next).finally(() => {
|
|
this.running = null;
|
|
this.child = null;
|
|
this.pump();
|
|
});
|
|
}
|
|
private async execute({ id, project }: { id: string; project: Project }) {
|
|
const j = this.jobs.get(id)!;
|
|
const folder = path.join(this.directory, id);
|
|
try {
|
|
j.status = "building";
|
|
await this.persist(j);
|
|
const bytes = await gameArchive(project, this.read);
|
|
if ((j.status as string) === "cancelled" || this.stopped) return;
|
|
const web = path.join(folder, "game");
|
|
await fs.mkdir(web, { recursive: true });
|
|
for (const [name, data] of Object.entries(unzipSync(bytes))) {
|
|
if (name.includes("..") || name.startsWith("/") || name.includes("\\"))
|
|
throw Error("Unsafe game path");
|
|
const file = path.join(web, name);
|
|
await fs.mkdir(path.dirname(file), { recursive: true });
|
|
await fs.writeFile(file, data);
|
|
}
|
|
await fs.writeFile(
|
|
path.join(folder, "build-config.json"),
|
|
JSON.stringify(j.options),
|
|
);
|
|
await fs.writeFile(
|
|
path.join(folder, "project-snapshot.json"),
|
|
JSON.stringify(project),
|
|
);
|
|
if ((j.status as string) === "cancelled" || this.stopped) return;
|
|
await new Promise<void>((resolve, reject) => {
|
|
const child = spawn(
|
|
process.execPath,
|
|
[
|
|
path.join(root, "native", "build.mjs"),
|
|
"--config",
|
|
path.join(folder, "build-config.json"),
|
|
"--game",
|
|
web,
|
|
"--out",
|
|
path.join(folder, "output"),
|
|
],
|
|
{
|
|
cwd: root,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
detached: process.platform !== "win32",
|
|
windowsHide: true,
|
|
},
|
|
);
|
|
this.child = child;
|
|
let remainder = "";
|
|
const append = (data: Buffer) => {
|
|
remainder += data.toString();
|
|
const lines = remainder.split(/[\r\n]+/);
|
|
remainder = lines.pop() || "";
|
|
for (const line of lines)
|
|
if (line) j.logs.push(this.redact(line).slice(0, 1000));
|
|
if (remainder.length > 10000) {
|
|
j.logs.push(this.redact(remainder).slice(0, 1000));
|
|
remainder = "";
|
|
}
|
|
j.logs = j.logs.slice(-500);
|
|
};
|
|
child.stdout?.on("data", append);
|
|
child.stderr?.on("data", append);
|
|
const timer = setTimeout(
|
|
() => {
|
|
j.error = "Build timed out after 20 minutes";
|
|
this.kill(child);
|
|
},
|
|
20 * 60 * 1000,
|
|
);
|
|
child.on("error", (e) => {
|
|
clearTimeout(timer);
|
|
reject(e);
|
|
});
|
|
child.on("close", (code) => {
|
|
clearTimeout(timer);
|
|
if (remainder) j.logs.push(this.redact(remainder).slice(0, 1000));
|
|
code === 0
|
|
? resolve()
|
|
: reject(Error(j.error || "Builder exited " + code));
|
|
});
|
|
});
|
|
if ((j.status as string) === "cancelled") return;
|
|
const manifest = JSON.parse(
|
|
await fs.readFile(
|
|
path.join(folder, "output", "build-manifest.json"),
|
|
"utf8",
|
|
),
|
|
);
|
|
j.artifacts = [];
|
|
for (const f of manifest.files) {
|
|
if (path.basename(f.name) !== f.name)
|
|
throw Error("Invalid build output");
|
|
const b = await fs.readFile(
|
|
path.join(folder, "output", "artifacts", f.name),
|
|
);
|
|
if (createHash("sha256").update(b).digest("hex") !== f.sha256)
|
|
throw Error("Artifact checksum mismatch");
|
|
j.artifacts.push({
|
|
...f,
|
|
path: path.join(folder, "output", "artifacts", f.name),
|
|
url: "/api/builds/" + id + "/files/" + encodeURIComponent(f.name),
|
|
});
|
|
}
|
|
if (!j.artifacts.length) throw Error("No artifacts");
|
|
j.status = "succeeded";
|
|
j.logs.push(
|
|
"Application package verified. Device execution is a separate check.",
|
|
);
|
|
} catch (e) {
|
|
if (j.status !== "cancelled") {
|
|
j.status = "failed";
|
|
j.error = this.redact(String(e));
|
|
j.logs.push(j.error);
|
|
}
|
|
} finally {
|
|
j.finishedAt = new Date().toISOString();
|
|
await this.persist(j);
|
|
}
|
|
}
|
|
private redact(s: string) {
|
|
for (const key of [
|
|
"FORMA_KEYSTORE_PASSWORD",
|
|
"FORMA_KEY_PASSWORD",
|
|
"FORMA_KEY_ALIAS",
|
|
"FORMA_KEYSTORE",
|
|
]) {
|
|
const value = process.env[key];
|
|
if (value) s = s.split(value).join("[redacted]");
|
|
}
|
|
return s.replace(/\x1b\[[0-9;]*m/g, "");
|
|
}
|
|
private kill(child: ChildProcess) {
|
|
const send = (signal: NodeJS.Signals) => {
|
|
try {
|
|
if (process.platform !== "win32" && child.pid)
|
|
process.kill(-child.pid, signal);
|
|
else child.kill(signal);
|
|
} catch {}
|
|
};
|
|
send("SIGTERM");
|
|
const timer = setTimeout(() => send("SIGKILL"), 4000);
|
|
timer.unref();
|
|
child.once("close", () => clearTimeout(timer));
|
|
}
|
|
async cancel(id: string) {
|
|
const j = this.jobs.get(id);
|
|
if (!j) throw Error("Build not found");
|
|
if (["queued", "building"].includes(j.status)) {
|
|
j.status = "cancelled";
|
|
j.finishedAt = new Date().toISOString();
|
|
this.queue = this.queue.filter((n) => n.id !== id);
|
|
if (this.running === id && this.child) this.kill(this.child);
|
|
await this.persist(j);
|
|
}
|
|
return this.get(id);
|
|
}
|
|
async artifact(id: string, name: string) {
|
|
const j = this.get(id);
|
|
if (j.status !== "succeeded" || !j.artifacts?.some((f) => f.name === name))
|
|
throw Error("Artifact not found");
|
|
return path.join(this.directory, id, "output", "artifacts", name);
|
|
}
|
|
async close() {
|
|
this.stopped = true;
|
|
for (const j of this.jobs.values())
|
|
if (["queued", "building"].includes(j.status)) await this.cancel(j.id);
|
|
await this.idle;
|
|
await this.saved;
|
|
}
|
|
}
|