Publish Forma Engine 0.3.0 source with documentation and CI
This commit is contained in:
emil28092005
@@ -0,0 +1,312 @@
|
||||
import { promises as fs } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { spawn, type ChildProcess } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { randomUUID, createHash } from "node:crypto";
|
||||
import { unzipSync } from "fflate";
|
||||
import { gameArchive } from "../engine/archive.ts";
|
||||
import { clone, type Project } from "../engine/schema.ts";
|
||||
import { normalizeOptions } from "../native/options.mjs";
|
||||
import { doctor } from "../native/build.mjs";
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
export type BuildJob = {
|
||||
id: string;
|
||||
status: "queued" | "building" | "succeeded" | "failed" | "cancelled";
|
||||
revision: number;
|
||||
projectId: string;
|
||||
createdAt: string;
|
||||
finishedAt?: string;
|
||||
options: ReturnType<typeof normalizeOptions>;
|
||||
logs: string[];
|
||||
error?: string;
|
||||
artifacts?: {
|
||||
name: string;
|
||||
bytes: number;
|
||||
sha256: string;
|
||||
url: string;
|
||||
path: string;
|
||||
}[];
|
||||
};
|
||||
export class BuildManager {
|
||||
private jobs = new Map<string, BuildJob>();
|
||||
private queue: { id: string; project: Project }[] = [];
|
||||
private child: ChildProcess | null = null;
|
||||
private running: string | null = null;
|
||||
private stopped = false;
|
||||
private idle: Promise<void> = Promise.resolve();
|
||||
private saved: Promise<void> = Promise.resolve();
|
||||
readonly directory: string;
|
||||
constructor(
|
||||
projectDir: string,
|
||||
private read: (uri: string) => Promise<Uint8Array>,
|
||||
) {
|
||||
this.directory = path.join(projectDir, "builds");
|
||||
}
|
||||
async init() {
|
||||
await fs.mkdir(this.directory, { recursive: true });
|
||||
for (const n of await fs.readdir(this.directory))
|
||||
if (/^[a-f0-9-]{36}$/.test(n)) {
|
||||
try {
|
||||
const j = JSON.parse(
|
||||
await fs.readFile(path.join(this.directory, n, "job.json"), "utf8"),
|
||||
) as BuildJob;
|
||||
if (j.id !== n) continue;
|
||||
if (["queued", "building"].includes(j.status)) {
|
||||
j.status = "failed";
|
||||
j.error = "Build interrupted by server restart";
|
||||
j.finishedAt = new Date().toISOString();
|
||||
await this.persist(j);
|
||||
}
|
||||
this.jobs.set(n, j);
|
||||
} catch {
|
||||
/* An unfinished metadata write must not prevent opening the project. */
|
||||
}
|
||||
}
|
||||
}
|
||||
capabilities() {
|
||||
return doctor();
|
||||
}
|
||||
list() {
|
||||
return [...this.jobs.values()]
|
||||
.sort((a, b) => b.createdAt.localeCompare(a.createdAt))
|
||||
.slice(0, 30)
|
||||
.map((j) => ({ ...j, logs: j.logs.slice(-80) }));
|
||||
}
|
||||
get(id: string) {
|
||||
const j = this.jobs.get(id);
|
||||
if (!j) throw Error("Build not found");
|
||||
return structuredClone(j);
|
||||
}
|
||||
private persist(job: BuildJob) {
|
||||
const data = JSON.stringify(job, null, 2),
|
||||
file = path.join(this.directory, job.id, "job.json");
|
||||
this.saved = this.saved
|
||||
.catch(() => {})
|
||||
.then(async () => {
|
||||
await fs.mkdir(path.dirname(file), { recursive: true });
|
||||
await fs.writeFile(file + ".tmp", data, { mode: 0o600 });
|
||||
await fs.rename(file + ".tmp", file);
|
||||
});
|
||||
return this.saved;
|
||||
}
|
||||
async start(project: Project, raw: any, expectedRevision: number) {
|
||||
if (this.stopped) throw Error("Build service closed");
|
||||
if (expectedRevision !== project.revision)
|
||||
throw Error("REVISION_CONFLICT: reread project before building");
|
||||
if (this.queue.length >= 3)
|
||||
throw Error("Build queue full (maximum 3 waiting jobs)");
|
||||
const options = normalizeOptions(raw),
|
||||
cap = await this.capabilities();
|
||||
if (!cap.targets[options.target as keyof typeof cap.targets].ready)
|
||||
throw Error(
|
||||
"BUILD_TOOLS_MISSING: " +
|
||||
cap.targets[options.target as keyof typeof cap.targets].missing.join(
|
||||
"; ",
|
||||
),
|
||||
);
|
||||
if (
|
||||
options.target === "android" &&
|
||||
options.mode === "release" &&
|
||||
!cap.targets.android.releaseSigningConfigured
|
||||
)
|
||||
throw Error(
|
||||
"RELEASE_SIGNING_MISSING: configure signing in the local server environment",
|
||||
);
|
||||
if (expectedRevision !== project.revision) throw Error("REVISION_CONFLICT");
|
||||
if (this.stopped) throw Error("Build service closed");
|
||||
if (this.queue.length >= 3) throw Error("Build queue full");
|
||||
const job: BuildJob = {
|
||||
id: randomUUID(),
|
||||
status: "queued",
|
||||
revision: project.revision,
|
||||
projectId: project.id,
|
||||
createdAt: new Date().toISOString(),
|
||||
options,
|
||||
logs: ["Snapshot revision " + project.revision],
|
||||
};
|
||||
this.jobs.set(job.id, job);
|
||||
this.queue.push({ id: job.id, project: clone(project) });
|
||||
await this.persist(job);
|
||||
this.pump();
|
||||
return this.get(job.id);
|
||||
}
|
||||
private pump() {
|
||||
if (this.running || this.stopped) return;
|
||||
const next = this.queue.shift();
|
||||
if (!next) return;
|
||||
this.running = next.id;
|
||||
this.idle = this.execute(next).finally(() => {
|
||||
this.running = null;
|
||||
this.child = null;
|
||||
this.pump();
|
||||
});
|
||||
}
|
||||
private async execute({ id, project }: { id: string; project: Project }) {
|
||||
const j = this.jobs.get(id)!;
|
||||
const folder = path.join(this.directory, id);
|
||||
try {
|
||||
j.status = "building";
|
||||
await this.persist(j);
|
||||
const bytes = await gameArchive(project, this.read);
|
||||
if ((j.status as string) === "cancelled" || this.stopped) return;
|
||||
const web = path.join(folder, "game");
|
||||
await fs.mkdir(web, { recursive: true });
|
||||
for (const [name, data] of Object.entries(unzipSync(bytes))) {
|
||||
if (name.includes("..") || name.startsWith("/") || name.includes("\\"))
|
||||
throw Error("Unsafe game path");
|
||||
const file = path.join(web, name);
|
||||
await fs.mkdir(path.dirname(file), { recursive: true });
|
||||
await fs.writeFile(file, data);
|
||||
}
|
||||
await fs.writeFile(
|
||||
path.join(folder, "build-config.json"),
|
||||
JSON.stringify(j.options),
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(folder, "project-snapshot.json"),
|
||||
JSON.stringify(project),
|
||||
);
|
||||
if ((j.status as string) === "cancelled" || this.stopped) return;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
path.join(root, "native", "build.mjs"),
|
||||
"--config",
|
||||
path.join(folder, "build-config.json"),
|
||||
"--game",
|
||||
web,
|
||||
"--out",
|
||||
path.join(folder, "output"),
|
||||
],
|
||||
{
|
||||
cwd: root,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
detached: process.platform !== "win32",
|
||||
windowsHide: true,
|
||||
},
|
||||
);
|
||||
this.child = child;
|
||||
let remainder = "";
|
||||
const append = (data: Buffer) => {
|
||||
remainder += data.toString();
|
||||
const lines = remainder.split(/[\r\n]+/);
|
||||
remainder = lines.pop() || "";
|
||||
for (const line of lines)
|
||||
if (line) j.logs.push(this.redact(line).slice(0, 1000));
|
||||
if (remainder.length > 10000) {
|
||||
j.logs.push(this.redact(remainder).slice(0, 1000));
|
||||
remainder = "";
|
||||
}
|
||||
j.logs = j.logs.slice(-500);
|
||||
};
|
||||
child.stdout?.on("data", append);
|
||||
child.stderr?.on("data", append);
|
||||
const timer = setTimeout(
|
||||
() => {
|
||||
j.error = "Build timed out after 20 minutes";
|
||||
this.kill(child);
|
||||
},
|
||||
20 * 60 * 1000,
|
||||
);
|
||||
child.on("error", (e) => {
|
||||
clearTimeout(timer);
|
||||
reject(e);
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
clearTimeout(timer);
|
||||
if (remainder) j.logs.push(this.redact(remainder).slice(0, 1000));
|
||||
code === 0
|
||||
? resolve()
|
||||
: reject(Error(j.error || "Builder exited " + code));
|
||||
});
|
||||
});
|
||||
if ((j.status as string) === "cancelled") return;
|
||||
const manifest = JSON.parse(
|
||||
await fs.readFile(
|
||||
path.join(folder, "output", "build-manifest.json"),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
j.artifacts = [];
|
||||
for (const f of manifest.files) {
|
||||
if (path.basename(f.name) !== f.name)
|
||||
throw Error("Invalid build output");
|
||||
const b = await fs.readFile(
|
||||
path.join(folder, "output", "artifacts", f.name),
|
||||
);
|
||||
if (createHash("sha256").update(b).digest("hex") !== f.sha256)
|
||||
throw Error("Artifact checksum mismatch");
|
||||
j.artifacts.push({
|
||||
...f,
|
||||
path: path.join(folder, "output", "artifacts", f.name),
|
||||
url: "/api/builds/" + id + "/files/" + encodeURIComponent(f.name),
|
||||
});
|
||||
}
|
||||
if (!j.artifacts.length) throw Error("No artifacts");
|
||||
j.status = "succeeded";
|
||||
j.logs.push(
|
||||
"Application package verified. Device execution is a separate check.",
|
||||
);
|
||||
} catch (e) {
|
||||
if (j.status !== "cancelled") {
|
||||
j.status = "failed";
|
||||
j.error = this.redact(String(e));
|
||||
j.logs.push(j.error);
|
||||
}
|
||||
} finally {
|
||||
j.finishedAt = new Date().toISOString();
|
||||
await this.persist(j);
|
||||
}
|
||||
}
|
||||
private redact(s: string) {
|
||||
for (const key of [
|
||||
"FORMA_KEYSTORE_PASSWORD",
|
||||
"FORMA_KEY_PASSWORD",
|
||||
"FORMA_KEY_ALIAS",
|
||||
"FORMA_KEYSTORE",
|
||||
]) {
|
||||
const value = process.env[key];
|
||||
if (value) s = s.split(value).join("[redacted]");
|
||||
}
|
||||
return s.replace(/\x1b\[[0-9;]*m/g, "");
|
||||
}
|
||||
private kill(child: ChildProcess) {
|
||||
const send = (signal: NodeJS.Signals) => {
|
||||
try {
|
||||
if (process.platform !== "win32" && child.pid)
|
||||
process.kill(-child.pid, signal);
|
||||
else child.kill(signal);
|
||||
} catch {}
|
||||
};
|
||||
send("SIGTERM");
|
||||
const timer = setTimeout(() => send("SIGKILL"), 4000);
|
||||
timer.unref();
|
||||
child.once("close", () => clearTimeout(timer));
|
||||
}
|
||||
async cancel(id: string) {
|
||||
const j = this.jobs.get(id);
|
||||
if (!j) throw Error("Build not found");
|
||||
if (["queued", "building"].includes(j.status)) {
|
||||
j.status = "cancelled";
|
||||
j.finishedAt = new Date().toISOString();
|
||||
this.queue = this.queue.filter((n) => n.id !== id);
|
||||
if (this.running === id && this.child) this.kill(this.child);
|
||||
await this.persist(j);
|
||||
}
|
||||
return this.get(id);
|
||||
}
|
||||
async artifact(id: string, name: string) {
|
||||
const j = this.get(id);
|
||||
if (j.status !== "succeeded" || !j.artifacts?.some((f) => f.name === name))
|
||||
throw Error("Artifact not found");
|
||||
return path.join(this.directory, id, "output", "artifacts", name);
|
||||
}
|
||||
async close() {
|
||||
this.stopped = true;
|
||||
for (const j of this.jobs.values())
|
||||
if (["queued", "building"].includes(j.status)) await this.cancel(j.id);
|
||||
await this.idle;
|
||||
await this.saved;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user