fix: cloud worker transfers fail after plain workspace results are staged (#129223)

* fix(gateway): preserve plain worker workspaces after result staging

* chore(gateway): prune obsolete worker snapshot assertion baseline

* chore: keep cloud worker release notes in pull request
This commit is contained in:
Peter Steinberger
2026-08-25 03:37:15 -07:00
committed by GitHub
parent 233ac850f0
commit a85dbdaf0d
3 changed files with 88 additions and 22 deletions
-1
View File
@@ -3116,7 +3116,6 @@ src/gateway/worker-environments/live-event-projection.ts 1
src/gateway/worker-environments/node-worker-tunnel.ts 6
src/gateway/worker-environments/node-worker-workspace-fallback.ts 2
src/gateway/worker-environments/node-workspace-transfer-service.ts 1
src/gateway/worker-environments/node-workspace-transfer-snapshot.ts 1
src/gateway/worker-environments/placement-state.ts 3
src/gateway/worker-environments/placement-store.ts 1
src/gateway/worker-environments/provider-lifecycle.ts 1
@@ -7,6 +7,7 @@ import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"
import { NODE_WORKER_WORKSPACE_EXEC_COMMAND } from "../../infra/node-commands.js";
import { invokeNodeWorkerSupervisorCommand } from "../../node-host/node-worker-supervisor-commands.js";
import { NodeWorkerWorkspaceRuntime } from "../../node-host/node-worker-workspace.js";
import { runCommandWithTimeout } from "../../process/exec.js";
import type { ResolvedGatewayAuth } from "../auth.js";
import { createGatewayHttpServer } from "../server-http.js";
import { createNodeWorkspaceTransferHttpCallback } from "./node-workspace-transfer-http.js";
@@ -113,6 +114,79 @@ function retryOrUploadStatus(retryStarted: Promise<void>, upload: Promise<unknow
}
describe("node workspace transfer service", () => {
it("keeps a plain workspace transferable after durable result staging initializes Git", async () => {
const root = tempDirs.make("node-workspace-transfer-unborn-git-");
const localPath = path.join(root, "workspace");
await fs.mkdir(localPath);
await fs.writeFile(path.join(localPath, "input.txt"), "gateway input\n");
const service = createNodeWorkspaceTransferService({
getOwner: () => ({
credential: {
ownerEpoch: 1,
expiresAtMs: Date.now() + 60_000,
sessionId: "session-unborn",
},
environment: {
ownerEpoch: 1,
attachedSessionIds: ["session-unborn"],
destroyRequestedAtMs: null,
state: "attached",
},
}),
temporaryRoot: path.join(root, "transfer-tmp"),
});
const request = {
environmentId: "environment-unborn",
ownerEpoch: 1,
sessionId: "session-unborn",
localPath,
isAuthorized: () => true,
};
const git = async (...args: string[]) => {
const result = await runCommandWithTimeout(["git", "-C", localPath, ...args], {
timeoutMs: 10_000,
});
expect(result.code).toBe(0);
return result.stdout.trim();
};
try {
const plain = await service.prepareSync({ ...request, generation: 1 });
expect(plain.snapshot.manifest.baseCommit).toBeNull();
await git("init", "--quiet", "--object-format=sha1");
const staged = await service.prepareSync({ ...request, generation: 2 });
expect(staged.snapshot.manifest.baseCommit).toBeNull();
expect(staged.snapshot.packPath).toBeUndefined();
expect(staged.snapshot.manifestRef).toBe(plain.snapshot.manifestRef);
expect(staged.snapshot.manifest.entries).toContainEqual(
expect.objectContaining({ path: "input.txt", type: "file" }),
);
await git("add", "input.txt");
await git(
"-c",
"user.name=Worker Transfer Test",
"-c",
"user.email=worker-transfer@example.invalid",
"commit",
"--quiet",
"-m",
"tracked workspace",
);
const committed = await service.prepareSync({ ...request, generation: 3 });
expect(committed.snapshot.manifest.baseCommit).toBe(await git("rev-parse", "HEAD"));
expect(committed.snapshot.packPath).toBeDefined();
await fs.writeFile(path.join(localPath, ".git", "HEAD"), "invalid HEAD\n");
await expect(service.prepareSync({ ...request, generation: 4 })).rejects.toThrow(
"Worker workspace sync failed",
);
} finally {
await service.closeAll();
}
});
it("streams a plain workspace to the node and accepts only its changed result blobs", async () => {
const root = tempDirs.make("node-workspace-transfer-service-");
const localPath = path.join(root, "gateway-workspace");
@@ -7,6 +7,7 @@ import {
type WorkerWorkspaceManifest,
} from "./workspace-manifest.js";
import { readActualWorkspaceManifest } from "./workspace-reconcile.js";
import { probeWorkspaceGitMode } from "./workspace-sync-helpers.js";
import {
createWorkspaceGitTransferList,
readWorkspaceTransferPaths,
@@ -23,39 +24,31 @@ export type NodeWorkspaceTransferSnapshot = {
packPath?: string;
};
async function successfulGit(root: string, args: string[]): Promise<string> {
const result = await runCommandWithTimeout(["git", "-C", root, ...args], {
timeoutMs: TRANSFER_TIMEOUT_MS,
maxOutputBytes: 256 * 1024,
maxCombinedOutputBytes: 512 * 1024,
baseEnv: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "" },
});
if (result.termination !== "exit" || result.code !== 0) {
throw new Error("Worker workspace Git inspection failed");
}
return result.stdout.trim();
}
export async function prepareNodeWorkspaceTransferSnapshot(params: {
localPath: string;
temporaryRoot: string;
signal?: AbortSignal;
}): Promise<NodeWorkspaceTransferSnapshot> {
const root = await fsp.realpath(params.localPath);
const gitAdmin = await fsp.lstat(path.join(root, ".git")).catch((error: unknown) => {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return undefined;
}
throw error;
const git = await probeWorkspaceGitMode({
localPath: root,
commandOptions: {
timeoutMs: TRANSFER_TIMEOUT_MS,
maxOutputBytes: 256 * 1024,
maxCombinedOutputBytes: 512 * 1024,
baseEnv: { ...process.env, GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "" },
signal: params.signal,
},
runTask: runCommandWithTimeout,
});
let baseCommit: string | null = null;
let includePaths: ReadonlySet<string> | undefined;
if (gitAdmin) {
const gitRoot = await fsp.realpath(await successfulGit(root, ["rev-parse", "--show-toplevel"]));
if (git.mode === "git") {
const gitRoot = await fsp.realpath(git.gitRoot);
if (gitRoot !== root) {
throw new Error("Worker git workspace sync requires the managed worktree root");
}
baseCommit = await successfulGit(root, ["rev-parse", "--verify", "HEAD"]);
baseCommit = git.baseCommit;
if (!/^[a-f0-9]{40}(?:[a-f0-9]{24})?$/u.test(baseCommit)) {
throw new Error("Worker workspace Git base is not a commit id");
}