fix(matrix): retire shared E2EE clients safely (#119570)

* fix(qa): bound Matrix E2EE client shutdown

Punchcard-Session: silver-valley-valley-dt

* fix(qa): drain Matrix decryptions before SDK shutdown

Punchcard-Session: silver-valley-valley-dt

* fix(matrix): retire shared E2EE clients safely

Punchcard-Session: silver-valley-valley-dt

* fix(matrix): satisfy lifecycle lint gate

Punchcard-Session: silver-valley-valley-dt
This commit is contained in:
Vincent Koc
2026-08-06 01:14:31 +08:00
committed by GitHub
parent 9ef94e4ef9
commit a3419d4a4b
30 changed files with 3109 additions and 1440 deletions
@@ -6,16 +6,15 @@ import {
expectOneOffSharedMatrixClient,
matrixClientResolverMocks,
primeMatrixClientResolverMocks,
setAcquiredMatrixClient,
} from "../client-resolver.test-helpers.js";
const resolveMatrixRoomIdMock = vi.fn();
const {
loadConfigMock,
getMatrixRuntimeMock,
getActiveMatrixClientMock,
acquireSharedMatrixClientMock,
releaseSharedClientInstanceMock,
sharedLeaseReleaseMock,
isBunRuntimeMock,
resolveMatrixAuthContextMock,
} = matrixClientResolverMocks;
@@ -26,20 +25,12 @@ vi.mock("../../runtime.js", () => ({
getMatrixRuntime: () => getMatrixRuntimeMock(),
}));
vi.mock("../active-client.js", () => ({
getActiveMatrixClient: getActiveMatrixClientMock,
}));
vi.mock("../client.js", () => ({
acquireSharedMatrixClient: acquireSharedMatrixClientMock,
isBunRuntime: () => isBunRuntimeMock(),
resolveMatrixAuthContext: resolveMatrixAuthContextMock,
}));
vi.mock("../client/shared.js", () => ({
releaseSharedClientInstance: (...args: unknown[]) => releaseSharedClientInstanceMock(...args),
}));
vi.mock("../send.js", () => ({
resolveMatrixRoomId: (...args: unknown[]) => resolveMatrixRoomIdMock(...args),
}));
@@ -58,16 +49,14 @@ describe("action client helpers", () => {
primeMatrixClientResolverMocks();
resolveMatrixRoomIdMock
.mockReset()
.mockImplementation(async (clientForTest, roomId: string) => roomId);
.mockImplementation(async (_client, roomId: string) => roomId);
});
afterEach(() => {
vi.unstubAllEnvs();
});
it("stops one-off shared clients when no active monitor client is registered", async () => {
vi.stubEnv("OPENCLAW_GATEWAY_PORT", "18799");
it("borrows and releases one-off action clients", async () => {
const result = await withResolvedActionClient(
{ cfg: TEST_CFG, accountId: "default" },
async () => "ok",
@@ -77,90 +66,51 @@ describe("action client helpers", () => {
expect(result).toBe("ok");
});
it("skips one-off room preparation when readiness is disabled", async () => {
it("forwards the transient retirement signal to action work", async () => {
const sharedClient = createMockMatrixClient();
const lease = setAcquiredMatrixClient(sharedClient);
await withResolvedActionClient(
{ cfg: TEST_CFG, accountId: "default" },
async (_client, abortSignal) => {
expect(abortSignal).toBe(lease.abortSignal);
},
);
});
it("skips preparation when readiness is disabled", async () => {
await withResolvedActionClient(
{ cfg: TEST_CFG, accountId: "default", readiness: "none" },
async () => {},
);
const sharedClient = await acquireSharedMatrixClientMock.mock.results[0]?.value;
expect(sharedClient.prepareForOneOff).not.toHaveBeenCalled();
expect(sharedClient.start).not.toHaveBeenCalled();
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "stop");
const lease = await acquireSharedMatrixClientMock.mock.results[0]?.value;
expect(lease.client.prepareForOneOff).not.toHaveBeenCalled();
expect(lease.client.start).not.toHaveBeenCalled();
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: "stop" });
});
it("starts one-off clients when started readiness is required", async () => {
it("starts through the lease and persists started action clients", async () => {
await withStartedActionClient({ cfg: TEST_CFG, accountId: "default" }, async () => {});
const sharedClient = await acquireSharedMatrixClientMock.mock.results[0]?.value;
expect(sharedClient.start).toHaveBeenCalledTimes(1);
expect(sharedClient.prepareForOneOff).not.toHaveBeenCalled();
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "persist");
});
it("reuses active monitor client when available", async () => {
const activeClient = createMockMatrixClient();
getActiveMatrixClientMock.mockReturnValue(activeClient);
const result = await withResolvedActionClient(
{ cfg: TEST_CFG, accountId: "default" },
async (client) => {
expect(client).toBe(activeClient);
return "ok";
},
);
expect(result).toBe("ok");
expect(acquireSharedMatrixClientMock).not.toHaveBeenCalled();
expect(activeClient["stop"]).not.toHaveBeenCalled();
});
it("starts active clients when started readiness is required", async () => {
const activeClient = createMockMatrixClient();
getActiveMatrixClientMock.mockReturnValue(activeClient);
await withStartedActionClient({ cfg: TEST_CFG, accountId: "default" }, async (client) => {
expect(client).toBe(activeClient);
await expectOneOffSharedMatrixClient({
prepareForOneOffCalls: 0,
startCalls: 1,
releaseMode: "persist",
});
expect(activeClient["start"]).toHaveBeenCalledTimes(1);
expect(activeClient["prepareForOneOff"]).not.toHaveBeenCalled();
expect(activeClient["stop"]).not.toHaveBeenCalled();
expect(activeClient["stopAndPersist"]).not.toHaveBeenCalled();
});
it("uses the implicit resolved account id for active client lookup and storage", async () => {
loadConfigMock.mockReturnValue({
channels: {
matrix: {
accounts: {
ops: {
homeserver: "https://ops.example.org",
userId: "@ops:example.org",
accessToken: "ops-token",
},
},
},
},
});
it("uses the implicit resolved account id for shared acquisition", async () => {
resolveMatrixAuthContextMock.mockReturnValue({
cfg: loadConfigMock(),
cfg: TEST_CFG,
env: process.env,
accountId: "ops",
resolved: {
homeserver: "https://ops.example.org",
userId: "@ops:example.org",
accessToken: "ops-token",
deviceId: "OPSDEVICE",
encryption: true,
},
resolved: {},
});
await withResolvedActionClient({ cfg: loadConfigMock() as never }, async () => {});
await expectOneOffSharedMatrixClient({
cfg: loadConfigMock(),
accountId: "ops",
});
await withResolvedActionClient({ cfg: TEST_CFG }, async () => {});
await expectOneOffSharedMatrixClient({ accountId: "ops" });
});
it("uses explicit cfg instead of loading runtime config", async () => {
@@ -180,42 +130,24 @@ describe("action client helpers", () => {
});
});
it("stops shared action clients after wrapped calls succeed", async () => {
it("releases shared action clients with the requested discard mode", async () => {
const sharedClient = createMockMatrixClient();
acquireSharedMatrixClientMock.mockResolvedValue(sharedClient);
setAcquiredMatrixClient(sharedClient);
const result = await withResolvedActionClient(
await withResolvedActionClient(
{ cfg: TEST_CFG, accountId: "default" },
async (client) => {
expect(client).toBe(sharedClient);
return "ok";
},
);
expect(result).toBe("ok");
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "stop");
});
it("can discard read-only shared action clients without persisting crypto state", async () => {
const sharedClient = createMockMatrixClient();
acquireSharedMatrixClientMock.mockResolvedValue(sharedClient);
const result = await withResolvedActionClient(
{ cfg: TEST_CFG, accountId: "default" },
async (client) => {
expect(client).toBe(sharedClient);
return "ok";
},
"discard",
);
expect(result).toBe("ok");
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "discard");
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: "discard" });
});
it("stops shared action clients when the wrapped call throws", async () => {
it("releases shared action clients when the wrapped call throws", async () => {
const sharedClient = createMockMatrixClient();
acquireSharedMatrixClientMock.mockResolvedValue(sharedClient);
setAcquiredMatrixClient(sharedClient);
await expect(
withResolvedActionClient({ cfg: TEST_CFG, accountId: "default" }, async () => {
@@ -223,25 +155,37 @@ describe("action client helpers", () => {
}),
).rejects.toThrow("boom");
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "stop");
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: "stop" });
});
it("does not borrow an explicitly injected action client", async () => {
const injected = createMockMatrixClient();
await withResolvedActionClient({ client: injected }, async (client) => {
expect(client).toBe(injected);
});
expect(acquireSharedMatrixClientMock).not.toHaveBeenCalled();
expect(sharedLeaseReleaseMock).not.toHaveBeenCalled();
});
it("resolves room ids before running wrapped room actions", async () => {
const sharedClient = createMockMatrixClient();
acquireSharedMatrixClientMock.mockResolvedValue(sharedClient);
const lease = setAcquiredMatrixClient(sharedClient);
resolveMatrixRoomIdMock.mockResolvedValue("!room:example.org");
const result = await withResolvedRoomAction(
"room:#ops:example.org",
{ cfg: TEST_CFG, accountId: "default" },
async (client, resolvedRoom) => {
async (client, resolvedRoom, abortSignal) => {
expect(client).toBe(sharedClient);
expect(abortSignal).toBe(lease.abortSignal);
return resolvedRoom;
},
);
expect(resolveMatrixRoomIdMock).toHaveBeenCalledWith(sharedClient, "room:#ops:example.org");
expect(result).toBe("!room:example.org");
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "stop");
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: "stop" });
});
});
@@ -7,7 +7,7 @@ type MatrixActionClientStopMode = "stop" | "persist" | "discard";
export async function withResolvedActionClient<T>(
opts: MatrixActionClientOpts,
run: (client: MatrixActionClient["client"]) => Promise<T>,
run: (client: MatrixActionClient["client"], abortSignal?: AbortSignal) => Promise<T>,
mode: MatrixActionClientStopMode = "stop",
): Promise<T> {
return await withResolvedRuntimeMatrixClient(opts, run, mode);
@@ -15,7 +15,7 @@ export async function withResolvedActionClient<T>(
export async function withStartedActionClient<T>(
opts: MatrixActionClientOpts,
run: (client: MatrixActionClient["client"]) => Promise<T>,
run: (client: MatrixActionClient["client"], abortSignal?: AbortSignal) => Promise<T>,
): Promise<T> {
return await withResolvedActionClient({ ...opts, readiness: "started" }, run, "persist");
}
@@ -23,10 +23,14 @@ export async function withStartedActionClient<T>(
export async function withResolvedRoomAction<T>(
roomId: string,
opts: MatrixActionClientOpts,
run: (client: MatrixActionClient["client"], resolvedRoom: string) => Promise<T>,
run: (
client: MatrixActionClient["client"],
resolvedRoom: string,
abortSignal?: AbortSignal,
) => Promise<T>,
): Promise<T> {
return await withResolvedActionClient(opts, async (client) => {
return await withResolvedActionClient(opts, async (client, abortSignal) => {
const resolvedRoom = await resolveMatrixRoomId(client, roomId);
return await run(client, resolvedRoom);
return await run(client, resolvedRoom, abortSignal);
});
}
@@ -1,27 +0,0 @@
// Matrix plugin module implements active client behavior.
import { DEFAULT_ACCOUNT_ID, normalizeAccountId } from "openclaw/plugin-sdk/account-id";
import type { MatrixClient } from "./sdk.js";
const activeClients = new Map<string, MatrixClient>();
function resolveAccountKey(accountId?: string | null): string {
const normalized = normalizeAccountId(accountId);
return normalized || DEFAULT_ACCOUNT_ID;
}
export function setActiveMatrixClient(
client: MatrixClient | null,
accountId?: string | null,
): void {
const key = resolveAccountKey(accountId);
if (!client) {
activeClients.delete(key);
return;
}
activeClients.set(key, client);
}
export function getActiveMatrixClient(accountId?: string | null): MatrixClient | null {
const key = resolveAccountKey(accountId);
return activeClients.get(key) ?? null;
}
@@ -4,13 +4,13 @@ import {
createMockMatrixClient,
matrixClientResolverMocks,
primeMatrixClientResolverMocks,
setAcquiredMatrixClient,
} from "./client-resolver.test-helpers.js";
const {
getMatrixRuntimeMock,
getActiveMatrixClientMock,
acquireSharedMatrixClientMock,
releaseSharedClientInstanceMock,
sharedLeaseReleaseMock,
isBunRuntimeMock,
resolveMatrixAuthContextMock,
} = matrixClientResolverMocks;
@@ -21,20 +21,12 @@ vi.mock("../runtime.js", () => ({
getMatrixRuntime: () => getMatrixRuntimeMock(),
}));
vi.mock("./active-client.js", () => ({
getActiveMatrixClient: (...args: unknown[]) => getActiveMatrixClientMock(...args),
}));
vi.mock("./client.js", () => ({
acquireSharedMatrixClient: (...args: unknown[]) => acquireSharedMatrixClientMock(...args),
isBunRuntime: () => isBunRuntimeMock(),
resolveMatrixAuthContext: resolveMatrixAuthContextMock,
}));
vi.mock("./client/shared.js", () => ({
releaseSharedClientInstance: (...args: unknown[]) => releaseSharedClientInstanceMock(...args),
}));
let resolveRuntimeMatrixClientWithReadiness: typeof import("./client-bootstrap.js").resolveRuntimeMatrixClientWithReadiness;
let withResolvedRuntimeMatrixClient: typeof import("./client-bootstrap.js").withResolvedRuntimeMatrixClient;
@@ -53,9 +45,10 @@ describe("client bootstrap", () => {
});
it("releases leased shared clients when readiness setup fails", async () => {
const sharedClient = createMockMatrixClient();
vi.mocked(sharedClient["prepareForOneOff"]).mockRejectedValue(new Error("prepare failed"));
acquireSharedMatrixClientMock.mockResolvedValue(sharedClient);
const prepareForOneOff = vi.fn(async () => undefined);
const sharedClient = Object.assign(createMockMatrixClient(), { prepareForOneOff });
prepareForOneOff.mockRejectedValue(new Error("prepare failed"));
setAcquiredMatrixClient(sharedClient);
await expect(
resolveRuntimeMatrixClientWithReadiness({
@@ -65,13 +58,14 @@ describe("client bootstrap", () => {
}),
).rejects.toThrow("prepare failed");
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "stop");
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: "stop" });
});
it("releases leased shared clients when the wrapped action throws during readiness", async () => {
const sharedClient = createMockMatrixClient();
vi.mocked(sharedClient["start"]).mockRejectedValue(new Error("start failed"));
acquireSharedMatrixClientMock.mockResolvedValue(sharedClient);
it("starts through the shared lease and releases when startup fails", async () => {
const start = vi.fn(async () => undefined);
const sharedClient = Object.assign(createMockMatrixClient(), { start });
start.mockRejectedValue(new Error("start failed"));
setAcquiredMatrixClient(sharedClient);
await expect(
withResolvedRuntimeMatrixClient(
@@ -84,6 +78,58 @@ describe("client bootstrap", () => {
),
).rejects.toThrow("start failed");
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "stop");
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: "stop" });
});
it("borrows every non-injected client from the shared owner", async () => {
const sharedClient = createMockMatrixClient();
setAcquiredMatrixClient(sharedClient);
await withResolvedRuntimeMatrixClient(
{ cfg: TEST_CFG, accountId: "default", readiness: "none" },
async (client) => {
expect(client).toBe(sharedClient);
},
"persist",
);
expect(acquireSharedMatrixClientMock).toHaveBeenCalledWith({
cfg: TEST_CFG,
timeoutMs: undefined,
accountId: "default",
startClient: false,
role: "transient",
});
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: "persist" });
});
it("passes the transient retirement signal to admitted work", async () => {
const sharedClient = createMockMatrixClient();
const lease = setAcquiredMatrixClient(sharedClient);
await withResolvedRuntimeMatrixClient(
{ cfg: TEST_CFG, accountId: "default", readiness: "none" },
async (client, abortSignal) => {
expect(client).toBe(sharedClient);
expect(abortSignal).toBe(lease.abortSignal);
},
);
});
it("does not borrow or stop an explicitly injected client", async () => {
const start = vi.fn(async () => undefined);
const injected = Object.assign(createMockMatrixClient(), { start });
await withResolvedRuntimeMatrixClient(
{ client: injected, readiness: "started" },
async (client) => {
expect(client).toBe(injected);
},
"persist",
);
expect(start).toHaveBeenCalledTimes(1);
expect(acquireSharedMatrixClientMock).not.toHaveBeenCalled();
expect(sharedLeaseReleaseMock).not.toHaveBeenCalled();
});
});
@@ -2,41 +2,37 @@ import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
// Matrix plugin module implements client bootstrap behavior.
import { requireRuntimeConfig } from "openclaw/plugin-sdk/plugin-config-runtime";
import type { CoreConfig } from "../types.js";
import { getActiveMatrixClient } from "./active-client.js";
import { isBunRuntime } from "./client/runtime.js";
import type { SharedMatrixClientLease } from "./client/shared.js";
import type { MatrixClient } from "./sdk.js";
type ResolvedRuntimeMatrixClient = {
client: MatrixClient;
stopOnDone: boolean;
cleanup?: (mode: ResolvedRuntimeMatrixClientStopMode) => Promise<void>;
lease?: SharedMatrixClientLease;
};
type MatrixRuntimeClientReadiness = "none" | "prepared" | "started";
type ResolvedRuntimeMatrixClientStopMode = "stop" | "persist" | "discard";
type MatrixResolvedClientHook = (
client: MatrixClient,
context: { preparedByDefault: boolean },
) => Promise<void> | void;
const loadMatrixSharedClientRuntimeDeps = createLazyRuntimeModule(() =>
Promise.all([import("./client.js"), import("./client/shared.js")]).then(
([clientModule, sharedModule]) => ({
acquireSharedMatrixClient: clientModule.acquireSharedMatrixClient,
resolveMatrixAuthContext: clientModule.resolveMatrixAuthContext,
releaseSharedClientInstance: sharedModule.releaseSharedClientInstance,
}),
),
import("./client.js").then((clientModule) => ({
acquireSharedMatrixClient: clientModule.acquireSharedMatrixClient,
resolveMatrixAuthContext: clientModule.resolveMatrixAuthContext,
})),
);
async function ensureResolvedClientReadiness(params: {
client: MatrixClient;
lease?: SharedMatrixClientLease;
readiness?: MatrixRuntimeClientReadiness;
preparedByDefault: boolean;
}): Promise<void> {
if (params.readiness === "started") {
await params.client.start();
if (params.lease) {
await params.lease.start();
} else {
await params.client.start();
}
return;
}
if (params.readiness === "prepared" || (!params.readiness && params.preparedByDefault)) {
@@ -50,17 +46,21 @@ function ensureMatrixNodeRuntime() {
}
}
async function resolveRuntimeMatrixClient(opts: {
export async function resolveRuntimeMatrixClientWithReadiness(opts: {
client?: MatrixClient;
cfg?: CoreConfig;
timeoutMs?: number;
accountId?: string | null;
onResolved?: MatrixResolvedClientHook;
readiness?: MatrixRuntimeClientReadiness;
}): Promise<ResolvedRuntimeMatrixClient> {
ensureMatrixNodeRuntime();
if (opts.client) {
await opts.onResolved?.(opts.client, { preparedByDefault: false });
return { client: opts.client, stopOnDone: false };
await ensureResolvedClientReadiness({
client: opts.client,
readiness: opts.readiness,
preparedByDefault: false,
});
return { client: opts.client };
}
if (!opts.cfg) {
@@ -69,82 +69,36 @@ async function resolveRuntimeMatrixClient(opts: {
);
}
const cfg = requireRuntimeConfig(opts.cfg, "Matrix runtime client") as CoreConfig;
const { acquireSharedMatrixClient, releaseSharedClientInstance, resolveMatrixAuthContext } =
const { acquireSharedMatrixClient, resolveMatrixAuthContext } =
await loadMatrixSharedClientRuntimeDeps();
const authContext = resolveMatrixAuthContext({
cfg,
accountId: opts.accountId,
});
const active = getActiveMatrixClient(authContext.accountId);
if (active) {
await opts.onResolved?.(active, { preparedByDefault: false });
return { client: active, stopOnDone: false };
}
const client = await acquireSharedMatrixClient({
const lease = await acquireSharedMatrixClient({
cfg,
timeoutMs: opts.timeoutMs,
accountId: authContext.accountId,
startClient: false,
role: "transient",
});
try {
await opts.onResolved?.(client, { preparedByDefault: true });
await ensureResolvedClientReadiness({
client: lease.client,
lease,
readiness: opts.readiness,
preparedByDefault: true,
});
} catch (err) {
await releaseSharedClientInstance(client, "stop");
await lease.release({ mode: "stop" });
throw err;
}
return {
client,
stopOnDone: true,
cleanup: async (mode) => {
await releaseSharedClientInstance(client, mode);
},
client: lease.client,
lease,
};
}
export async function resolveRuntimeMatrixClientWithReadiness(opts: {
client?: MatrixClient;
cfg?: CoreConfig;
timeoutMs?: number;
accountId?: string | null;
readiness?: MatrixRuntimeClientReadiness;
}): Promise<ResolvedRuntimeMatrixClient> {
return await resolveRuntimeMatrixClient({
client: opts.client,
cfg: opts.cfg,
timeoutMs: opts.timeoutMs,
accountId: opts.accountId,
onResolved: async (client, context) => {
await ensureResolvedClientReadiness({
client,
readiness: opts.readiness,
preparedByDefault: context.preparedByDefault,
});
},
});
}
async function stopResolvedRuntimeMatrixClient(
resolved: ResolvedRuntimeMatrixClient,
mode: ResolvedRuntimeMatrixClientStopMode = "stop",
): Promise<void> {
if (!resolved.stopOnDone) {
return;
}
if (resolved.cleanup) {
await resolved.cleanup(mode);
return;
}
if (mode === "persist") {
await resolved.client.stopAndPersist();
return;
}
if (mode === "discard") {
resolved.client.stopWithoutPersist();
return;
}
resolved.client.stop();
}
export async function withResolvedRuntimeMatrixClient<T>(
opts: {
client?: MatrixClient;
@@ -153,13 +107,13 @@ export async function withResolvedRuntimeMatrixClient<T>(
accountId?: string | null;
readiness?: MatrixRuntimeClientReadiness;
},
run: (client: MatrixClient) => Promise<T>,
run: (client: MatrixClient, abortSignal?: AbortSignal) => Promise<T>,
stopMode: ResolvedRuntimeMatrixClientStopMode = "stop",
): Promise<T> {
const resolved = await resolveRuntimeMatrixClientWithReadiness(opts);
try {
return await run(resolved.client);
return await run(resolved.client, resolved.lease?.abortSignal);
} finally {
await stopResolvedRuntimeMatrixClient(resolved, stopMode);
await resolved.lease?.release({ mode: stopMode });
}
}
@@ -1,13 +1,14 @@
// Matrix helper module supports client resolver helpers behavior.
import { expect, vi, type Mock } from "vitest";
import type { SharedMatrixClientLease } from "./client/shared.js";
import type { MatrixClient } from "./sdk.js";
type MatrixClientResolverMocks = {
loadConfigMock: Mock<() => unknown>;
getMatrixRuntimeMock: Mock<() => unknown>;
getActiveMatrixClientMock: Mock<(...args: unknown[]) => MatrixClient | null>;
acquireSharedMatrixClientMock: Mock<(...args: unknown[]) => Promise<MatrixClient>>;
releaseSharedClientInstanceMock: Mock<(...args: unknown[]) => Promise<boolean>>;
acquireSharedMatrixClientMock: Mock<(...args: unknown[]) => Promise<SharedMatrixClientLease>>;
sharedLeaseReleaseMock: Mock<(...args: unknown[]) => Promise<void>>;
sharedLeaseStartMock: Mock<(...args: unknown[]) => Promise<void>>;
isBunRuntimeMock: Mock<() => boolean>;
resolveMatrixAuthContextMock: Mock<
(params: { cfg: unknown; accountId?: string | null }) => unknown
@@ -17,9 +18,9 @@ type MatrixClientResolverMocks = {
export const matrixClientResolverMocks: MatrixClientResolverMocks = {
loadConfigMock: vi.fn(() => ({})),
getMatrixRuntimeMock: vi.fn(),
getActiveMatrixClientMock: vi.fn(),
acquireSharedMatrixClientMock: vi.fn(),
releaseSharedClientInstanceMock: vi.fn(),
sharedLeaseReleaseMock: vi.fn(),
sharedLeaseStartMock: vi.fn(),
isBunRuntimeMock: vi.fn(() => false),
resolveMatrixAuthContextMock: vi.fn(),
};
@@ -49,6 +50,24 @@ export function createMockMatrixClient(): MatrixClient {
} as unknown as MatrixClient;
}
export function setAcquiredMatrixClient(client: MatrixClient): SharedMatrixClientLease {
const { acquireSharedMatrixClientMock, sharedLeaseReleaseMock, sharedLeaseStartMock } =
matrixClientResolverMocks;
sharedLeaseStartMock.mockImplementation(async () => {
await client.start();
});
const lease: SharedMatrixClientLease = {
abortSignal: new AbortController().signal,
client,
role: "transient",
registerMonitorRetirement: vi.fn(),
start: sharedLeaseStartMock,
release: sharedLeaseReleaseMock,
};
acquireSharedMatrixClientMock.mockResolvedValue(lease);
return lease;
}
export function primeMatrixClientResolverMocks(params?: {
cfg?: unknown;
accountId?: string;
@@ -59,9 +78,9 @@ export function primeMatrixClientResolverMocks(params?: {
const {
loadConfigMock,
getMatrixRuntimeMock,
getActiveMatrixClientMock,
acquireSharedMatrixClientMock,
releaseSharedClientInstanceMock,
sharedLeaseReleaseMock,
sharedLeaseStartMock,
isBunRuntimeMock,
resolveMatrixAuthContextMock,
} = matrixClientResolverMocks;
@@ -85,9 +104,9 @@ export function primeMatrixClientResolverMocks(params?: {
current: loadConfigMock,
},
});
getActiveMatrixClientMock.mockReturnValue(null);
isBunRuntimeMock.mockReturnValue(false);
releaseSharedClientInstanceMock.mockReset().mockResolvedValue(true);
sharedLeaseReleaseMock.mockReset().mockResolvedValue(undefined);
sharedLeaseStartMock.mockReset();
resolveMatrixAuthContextMock.mockImplementation(
({
cfg: explicitCfg,
@@ -105,7 +124,8 @@ export function primeMatrixClientResolverMocks(params?: {
},
}),
);
acquireSharedMatrixClientMock.mockResolvedValue(client);
acquireSharedMatrixClientMock.mockReset();
setAcquiredMatrixClient(client);
return client;
}
@@ -118,31 +138,27 @@ export async function expectOneOffSharedMatrixClient(params?: {
startCalls?: number;
releaseMode?: "persist" | "stop" | "discard";
}) {
const {
getActiveMatrixClientMock,
acquireSharedMatrixClientMock,
releaseSharedClientInstanceMock,
} = matrixClientResolverMocks;
const { acquireSharedMatrixClientMock, sharedLeaseReleaseMock } = matrixClientResolverMocks;
const accountId = params?.accountId ?? "default";
const prepareForOneOffCalls = params?.prepareForOneOffCalls ?? 1;
const startCalls = params?.startCalls ?? 0;
const releaseMode = params?.releaseMode ?? "stop";
expect(getActiveMatrixClientMock).toHaveBeenCalledWith(accountId);
expect(acquireSharedMatrixClientMock).toHaveBeenCalledTimes(1);
expect(acquireSharedMatrixClientMock).toHaveBeenCalledWith({
cfg: params?.cfg ?? {},
timeoutMs: params?.timeoutMs,
accountId,
startClient: false,
role: "transient",
});
const sharedClient = await acquireSharedMatrixClientMock.mock.results[0]?.value;
expect(sharedClient.prepareForOneOff).toHaveBeenCalledTimes(prepareForOneOffCalls);
expect(sharedClient.start).toHaveBeenCalledTimes(startCalls);
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, releaseMode);
const lease = await acquireSharedMatrixClientMock.mock.results[0]?.value;
expect(lease.client.prepareForOneOff).toHaveBeenCalledTimes(prepareForOneOffCalls);
expect(lease.client.start).toHaveBeenCalledTimes(startCalls);
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: releaseMode });
return sharedClient;
return lease.client;
}
export function expectExplicitMatrixClientConfig(params: { cfg: unknown; accountId?: string }) {
@@ -160,5 +176,6 @@ export function expectExplicitMatrixClientConfig(params: { cfg: unknown; account
timeoutMs: undefined,
accountId,
startClient: false,
role: "transient",
});
}
+6 -7
View File
@@ -14,11 +14,10 @@ export {
validateMatrixHomeserverUrl,
} from "./client/config.js";
export { createMatrixClient } from "./client/create-client.js";
export {
acquireSharedMatrixClient,
removeSharedClientInstance,
releaseSharedClientInstance,
resolveSharedMatrixClient,
stopSharedClientForAccount,
stopSharedClientInstance,
export { acquireSharedMatrixClient, stopSharedClientForAccount } from "./client/shared.js";
export type {
MatrixClientLeaseRole,
MatrixClientReleaseMode,
MatrixMonitorRetirement,
SharedMatrixClientLease,
} from "./client/shared.js";
@@ -250,6 +250,51 @@ describe("SqliteBackedMatrixSyncStore", () => {
await expect(afterNewSync.getSavedSyncToken()).resolves.toBe("s456");
});
it("freezes the last admitted cursor and marks only that cursor clean", async () => {
const storageRoot = createStorageRoot();
const store = new SqliteBackedMatrixSyncStore(storageRoot);
await store.setSyncData(createSyncResponse("before-freeze"));
await store.freezeSyncCursorPersistence();
await store.setSyncData(createSyncResponse("after-freeze"));
store.markCleanShutdown();
await store.flush();
const persisted = new SqliteBackedMatrixSyncStore(storageRoot);
await expect(persisted.getSavedSyncToken()).resolves.toBe("before-freeze");
expect(persisted.hasSavedSyncFromCleanShutdown()).toBe(true);
});
it("waits for an in-flight pre-freeze persist before freezing the cursor", async () => {
const storageRoot = createStorageRoot();
const store = new SqliteBackedMatrixSyncStore(storageRoot);
await store.setSyncData(createSyncResponse("in-flight"));
const flush = store.flush();
const freeze = store.freezeSyncCursorPersistence();
await Promise.all([flush, freeze]);
store.markCleanShutdown();
await store.flush();
const persisted = new SqliteBackedMatrixSyncStore(storageRoot);
await expect(persisted.getSavedSyncToken()).resolves.toBe("in-flight");
expect(persisted.hasSavedSyncFromCleanShutdown()).toBe(true);
});
it("discards pending cursor writes without marking a poisoned shutdown clean", async () => {
const storageRoot = createStorageRoot();
const store = new SqliteBackedMatrixSyncStore(storageRoot);
await store.setSyncData(createSyncResponse("suspect"));
await store.freezeSyncCursorPersistence();
store.discardPendingSyncCursorPersistence();
await store.flush();
const persisted = new SqliteBackedMatrixSyncStore(storageRoot);
expect(persisted.hasSavedSync()).toBe(false);
expect(persisted.hasSavedSyncFromCleanShutdown()).toBe(false);
});
it("coalesces background persistence until the debounce window elapses", async () => {
vi.useFakeTimers();
const storageRoot = createStorageRoot();
@@ -166,6 +166,7 @@ export class SqliteBackedMatrixSyncStore extends MemoryStore {
private readonly hadCleanShutdownOnLoad: boolean;
private cleanShutdown = false;
private dirty = false;
private frozen = false;
private persistTimer: NodeJS.Timeout | null = null;
private persistPromise: Promise<void> | null = null;
@@ -225,6 +226,9 @@ export class SqliteBackedMatrixSyncStore extends MemoryStore {
}
override setSyncData(syncData: ISyncResponse): Promise<void> {
if (this.frozen) {
return Promise.resolve();
}
this.accumulator.accumulate(syncData);
this.savedSync = this.accumulator.getJSON();
this.markDirtyAndSchedulePersist();
@@ -238,6 +242,9 @@ export class SqliteBackedMatrixSyncStore extends MemoryStore {
}
override storeClientOptions(options: IStoredClientOpts) {
if (this.frozen) {
return Promise.resolve();
}
this.savedClientOptions = cloneJson(options);
void super.storeClientOptions(options);
this.markDirtyAndSchedulePersist();
@@ -285,6 +292,25 @@ export class SqliteBackedMatrixSyncStore extends MemoryStore {
this.dirty = true;
}
async freezeSyncCursorPersistence(): Promise<void> {
this.frozen = true;
if (this.persistTimer) {
clearTimeout(this.persistTimer);
this.persistTimer = null;
}
await this.persistPromise;
}
discardPendingSyncCursorPersistence(): void {
this.frozen = true;
if (this.persistTimer) {
clearTimeout(this.persistTimer);
this.persistTimer = null;
}
this.cleanShutdown = false;
this.dirty = false;
}
async flush(): Promise<void> {
if (this.persistTimer) {
clearTimeout(this.persistTimer);
@@ -301,6 +327,9 @@ export class SqliteBackedMatrixSyncStore extends MemoryStore {
}
private markDirtyAndSchedulePersist(): void {
if (this.frozen) {
return;
}
this.cleanShutdown = false;
this.dirty = true;
if (this.persistTimer) {
+604 -278
View File
@@ -18,11 +18,7 @@ vi.mock("./create-client.js", () => ({
}));
let acquireSharedMatrixClient: typeof import("./shared.js").acquireSharedMatrixClient;
let releaseSharedClientInstance: typeof import("./shared.js").releaseSharedClientInstance;
let resolveSharedMatrixClient: typeof import("./shared.js").resolveSharedMatrixClient;
let stopSharedClient: typeof import("./shared.js").stopSharedClient;
let stopSharedClientForAccount: typeof import("./shared.js").stopSharedClientForAccount;
let stopSharedClientInstance: typeof import("./shared.js").stopSharedClientInstance;
function authFor(accountId: string): MatrixAuth {
return {
@@ -38,83 +34,71 @@ function authFor(accountId: string): MatrixAuth {
};
}
async function expectMatrixStartupAbort(promise: Promise<unknown>): Promise<void> {
let rejection: unknown;
try {
await promise;
} catch (error) {
rejection = error;
}
expect(rejection).toBeInstanceOf(Error);
const error = rejection as Error;
expect(error.name).toBe("AbortError");
expect(error.message).toBe("Matrix startup aborted");
function createDeferred<T = void>() {
let resolve: (value: T | PromiseLike<T>) => void = () => {};
let reject: (reason?: unknown) => void = () => {};
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
resolve = resolvePromise;
reject = rejectPromise;
});
return { promise, reject, resolve };
}
function createMockClient(name: string) {
const client = {
function createMockClient(name: string, callOrder: string[] = []) {
return {
name,
start: vi.fn(async () => undefined),
stop: vi.fn(() => undefined),
start: vi.fn(async (_params?: { abortSignal?: AbortSignal }) => {
callOrder.push("start");
}),
quiesceSync: vi.fn(async () => {
callOrder.push("quiesce");
}),
stop: vi.fn(() => {
callOrder.push("stop");
}),
stopAndPersist: vi.fn(async () => {
callOrder.push("persist");
}),
stopWithoutPersist: vi.fn(() => {
callOrder.push("discard");
}),
drainPendingDecryptions: vi.fn(async (reason: string) => {
callOrder.push(
reason === "matrix monitor sync quiesce"
? "drain-quiesce"
: reason === "matrix shared client final shutdown"
? "drain-final"
: "drain-poison",
);
}),
getJoinedRooms: vi.fn(async () => [] as string[]),
crypto: undefined,
};
return client;
}
function primeAccountClientMocks(params?: {
mainAuth?: MatrixAuth;
opsAuth?: MatrixAuth;
mainClient?: ReturnType<typeof createMockClient>;
opsClient?: ReturnType<typeof createMockClient>;
}) {
const mainAuth = params?.mainAuth ?? authFor("main");
const opsAuth = params?.opsAuth ?? authFor("ops");
const mainClient = params?.mainClient ?? createMockClient("main");
const opsClient = params?.opsClient ?? createMockClient("ops");
resolveMatrixAuthMock.mockImplementation(async ({ accountId }: { accountId?: string }) =>
accountId === "ops" ? opsAuth : mainAuth,
);
createMatrixClientMock.mockImplementation(async ({ accountId }: { accountId?: string }) => {
if (accountId === "ops") {
return opsClient;
}
return mainClient;
});
return { mainAuth, opsAuth, mainClient, opsClient };
}
function createPendingSharedStartup(mainAuth = authFor("main")) {
let resolveStartup: (() => void) | undefined;
const mainClient = {
...createMockClient("main"),
start: vi.fn(
async () =>
await new Promise<void>((resolve) => {
resolveStartup = resolve;
}),
),
function createMonitorRetirement(callOrder: string[]) {
return {
closeTaskAdmission: vi.fn(() => callOrder.push("close-admission")),
detachListeners: vi.fn(() => callOrder.push("detach-listeners")),
waitForTasks: vi.fn(async () => {
callOrder.push("wait-tasks");
}),
cleanup: vi.fn(async () => {
callOrder.push("monitor-cleanup");
}),
};
resolveMatrixAuthMock.mockResolvedValue(mainAuth);
createMatrixClientMock.mockResolvedValue(mainClient);
return { mainClient, resolveStartup: () => resolveStartup?.() };
}
describe("resolveSharedMatrixClient", () => {
async function expectMatrixStartupAbort(promise: Promise<unknown>): Promise<void> {
await expect(promise).rejects.toMatchObject({
name: "AbortError",
message: "Matrix startup aborted",
});
}
describe("shared Matrix client generations", () => {
beforeAll(async () => {
({
acquireSharedMatrixClient,
releaseSharedClientInstance,
resolveSharedMatrixClient,
stopSharedClient,
stopSharedClientForAccount,
stopSharedClientInstance,
} = await import("./shared.js"));
({ acquireSharedMatrixClient, stopSharedClientForAccount } = await import("./shared.js"));
});
beforeEach(() => {
@@ -131,262 +115,604 @@ describe("resolveSharedMatrixClient", () => {
);
});
afterEach(() => {
stopSharedClient();
afterEach(async () => {
await Promise.allSettled([
stopSharedClientForAccount(authFor("main")),
stopSharedClientForAccount(authFor("ops")),
]);
vi.useRealTimers();
vi.clearAllMocks();
});
it("keeps account clients isolated when resolves are interleaved", async () => {
const { mainClient, opsClient } = primeAccountClientMocks();
const firstMain = await resolveSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
startClient: false,
});
const firstPoe = await resolveSharedMatrixClient({
cfg: TEST_CFG,
accountId: "ops",
startClient: false,
});
const secondMain = await resolveSharedMatrixClient({ cfg: TEST_CFG, accountId: "main" });
expect(firstMain).toBe(mainClient);
expect(firstPoe).toBe(opsClient);
expect(secondMain).toBe(mainClient);
expect(createMatrixClientMock).toHaveBeenCalledTimes(2);
expect(mainClient.start).toHaveBeenCalledTimes(1);
expect(opsClient.start).toHaveBeenCalledTimes(0);
});
it("stops only the targeted account client", async () => {
const { mainAuth, mainClient, opsClient } = primeAccountClientMocks();
await resolveSharedMatrixClient({ cfg: TEST_CFG, accountId: "main", startClient: false });
await resolveSharedMatrixClient({ cfg: TEST_CFG, accountId: "ops", startClient: false });
stopSharedClientForAccount(mainAuth);
expect(mainClient.stop).toHaveBeenCalledTimes(1);
expect(opsClient.stop).toHaveBeenCalledTimes(0);
stopSharedClient();
expect(opsClient.stop).toHaveBeenCalledTimes(1);
});
it("drops stopped shared clients by instance so the next resolve recreates them", async () => {
const mainAuth = authFor("main");
const firstMainClient = createMockClient("main-first");
const secondMainClient = createMockClient("main-second");
resolveMatrixAuthMock.mockResolvedValue(mainAuth);
createMatrixClientMock
.mockResolvedValueOnce(firstMainClient)
.mockResolvedValueOnce(secondMainClient);
const first = await resolveSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
startClient: false,
});
stopSharedClientInstance(first as unknown as import("../sdk.js").MatrixClient);
const second = await resolveSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
startClient: false,
});
expect(first).toBe(firstMainClient);
expect(second).toBe(secondMainClient);
expect(firstMainClient.stop).toHaveBeenCalledTimes(1);
expect(createMatrixClientMock).toHaveBeenCalledTimes(2);
});
it("reuses the effective implicit account instead of keying it as default", async () => {
const poeAuth = authFor("ops");
const poeClient = createMockClient("ops");
resolveMatrixAuthContextMock.mockReturnValue({
cfg: TEST_CFG,
env: undefined,
accountId: "ops",
resolved: {},
});
resolveMatrixAuthMock.mockResolvedValue(poeAuth);
createMatrixClientMock.mockResolvedValue(poeClient);
const first = await resolveSharedMatrixClient({ cfg: TEST_CFG, startClient: false });
const second = await resolveSharedMatrixClient({ cfg: TEST_CFG, startClient: false });
expect(first).toBe(poeClient);
expect(second).toBe(poeClient);
expect(resolveMatrixAuthMock).toHaveBeenCalledWith({
cfg: TEST_CFG,
env: undefined,
accountId: "ops",
});
expect(createMatrixClientMock).toHaveBeenCalledTimes(1);
expect(createMatrixClientMock).toHaveBeenCalledWith({
accessToken: "token-ops",
accountId: "ops",
allowPrivateNetwork: undefined,
deviceId: "OPS-DEVICE",
dispatcherPolicy: undefined,
encryption: false,
homeserver: "https://matrix.example.org",
initialSyncLimit: undefined,
localTimeoutMs: undefined,
password: "secret",
ssrfPolicy: undefined,
userId: "@ops:example.org",
});
});
it("honors startClient false even when the caller acquires a shared lease", async () => {
const mainAuth = authFor("main");
it("keeps account generations isolated", async () => {
const mainClient = createMockClient("main");
const opsClient = createMockClient("ops");
resolveMatrixAuthMock.mockImplementation(async ({ accountId }: { accountId?: string }) =>
accountId === "ops" ? authFor("ops") : authFor("main"),
);
createMatrixClientMock.mockImplementation(async ({ accountId }: { accountId?: string }) =>
accountId === "ops" ? opsClient : mainClient,
);
resolveMatrixAuthMock.mockResolvedValue(mainAuth);
createMatrixClientMock.mockResolvedValue(mainClient);
const client = await acquireSharedMatrixClient({
const main = await acquireSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
startClient: false,
});
const ops = await acquireSharedMatrixClient({
cfg: TEST_CFG,
accountId: "ops",
startClient: false,
});
const secondMain = await acquireSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
startClient: false,
});
expect(client).toBe(mainClient);
expect(mainClient.start).not.toHaveBeenCalled();
expect(main.client).toBe(mainClient);
expect(ops.client).toBe(opsClient);
expect(secondMain.client).toBe(mainClient);
expect(createMatrixClientMock).toHaveBeenCalledTimes(2);
await Promise.all([main.release(), secondMain.release(), ops.release()]);
});
it("keeps shared clients alive until the last one-off lease releases", async () => {
it("retires only the requested account through the owner", async () => {
const mainAuth = authFor("main");
const mainClient = {
...createMockClient("main"),
stopAndPersist: vi.fn(async () => undefined),
};
const opsAuth = authFor("ops");
const mainClient = createMockClient("main");
const opsClient = createMockClient("ops");
createMatrixClientMock.mockResolvedValueOnce(mainClient).mockResolvedValueOnce(opsClient);
resolveMatrixAuthMock.mockResolvedValue(mainAuth);
createMatrixClientMock.mockResolvedValue(mainClient);
await acquireSharedMatrixClient({ auth: mainAuth, startClient: false });
const ops = await acquireSharedMatrixClient({ auth: opsAuth, startClient: false });
await stopSharedClientForAccount(mainAuth);
expect(mainClient.stopAndPersist).toHaveBeenCalledTimes(1);
expect(mainClient.stopWithoutPersist).not.toHaveBeenCalled();
expect(opsClient.stopWithoutPersist).not.toHaveBeenCalled();
await ops.release();
});
it("runs registered monitor cleanup during forced account retirement", async () => {
const auth = authFor("main");
const client = createMockClient("main");
const waitForTasks = createDeferred();
createMatrixClientMock.mockResolvedValue(client);
const monitor = await acquireSharedMatrixClient({
auth,
role: "monitor",
startClient: false,
});
const retirement = createMonitorRetirement([]);
retirement.waitForTasks.mockReturnValue(waitForTasks.promise);
monitor.registerMonitorRetirement(retirement);
const forcedRetirement = stopSharedClientForAccount(auth);
await vi.waitFor(() => {
expect(retirement.waitForTasks).toHaveBeenCalledTimes(1);
});
const lateRelease = monitor.release({ mode: "persist" });
expect(monitor.release({ mode: "discard" })).toBe(lateRelease);
let lateReleaseSettled = false;
void lateRelease.then(() => {
lateReleaseSettled = true;
});
await Promise.resolve();
expect(lateReleaseSettled).toBe(false);
waitForTasks.resolve();
await Promise.all([forcedRetirement, lateRelease]);
expect(retirement.closeTaskAdmission).toHaveBeenCalledTimes(1);
expect(retirement.detachListeners).toHaveBeenCalledTimes(1);
expect(retirement.waitForTasks).toHaveBeenCalledTimes(1);
expect(retirement.cleanup).toHaveBeenCalledTimes(1);
expect(client.quiesceSync).toHaveBeenCalledTimes(1);
expect(client.stopAndPersist).toHaveBeenCalledTimes(1);
});
it("runs every monitor cleanup once during forced account retirement", async () => {
const client = createMockClient("main");
createMatrixClientMock.mockResolvedValue(client);
const auth = authFor("main");
const first = await acquireSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
auth,
role: "monitor",
startClient: false,
});
const second = await acquireSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
auth,
role: "monitor",
startClient: false,
});
const firstRetirement = createMonitorRetirement([]);
const secondRetirement = createMonitorRetirement([]);
first.registerMonitorRetirement(firstRetirement);
second.registerMonitorRetirement(secondRetirement);
await stopSharedClientForAccount(auth);
await Promise.all([first.release(), second.release()]);
for (const retirement of [firstRetirement, secondRetirement]) {
expect(retirement.closeTaskAdmission).toHaveBeenCalledTimes(1);
expect(retirement.detachListeners).toHaveBeenCalledTimes(1);
expect(retirement.waitForTasks).toHaveBeenCalledTimes(1);
expect(retirement.cleanup).toHaveBeenCalledTimes(1);
}
expect(client.quiesceSync).toHaveBeenCalledTimes(1);
expect(client.stopAndPersist).toHaveBeenCalledTimes(1);
expect(client.stopWithoutPersist).not.toHaveBeenCalled();
});
it("reuses one generation for monitor and transient leases when transient releases first", async () => {
const callOrder: string[] = [];
const client = createMockClient("main", callOrder);
createMatrixClientMock.mockResolvedValue(client);
const auth = authFor("main");
const monitor = await acquireSharedMatrixClient({
auth,
role: "monitor",
startClient: false,
});
const transient = await acquireSharedMatrixClient({
auth,
role: "transient",
startClient: false,
});
expect(first).toBe(mainClient);
expect(second).toBe(mainClient);
expect(transient.abortSignal.aborted).toBe(false);
await transient.release({ mode: "stop" });
expect(transient.abortSignal.aborted).toBe(false);
expect(callOrder).toEqual([]);
expect(
await releaseSharedClientInstance(mainClient as unknown as import("../sdk.js").MatrixClient),
).toBe(false);
expect(mainClient.stop).not.toHaveBeenCalled();
monitor.registerMonitorRetirement(createMonitorRetirement(callOrder));
await monitor.release({ mode: "persist" });
expect(
await releaseSharedClientInstance(mainClient as unknown as import("../sdk.js").MatrixClient),
).toBe(true);
expect(mainClient.stop).toHaveBeenCalledTimes(1);
expect(callOrder).toEqual([
"quiesce",
"drain-quiesce",
"close-admission",
"detach-listeners",
"wait-tasks",
"monitor-cleanup",
"drain-final",
"persist",
]);
});
it("rejects mismatched explicit account ids when auth is already resolved", async () => {
await expect(
resolveSharedMatrixClient({
auth: authFor("ops"),
accountId: "main",
startClient: false,
}),
).rejects.toThrow("Matrix shared client account mismatch");
it("signals cooperative transient work and persists after it drains", async () => {
const callOrder: string[] = [];
const client = createMockClient("main", callOrder);
createMatrixClientMock.mockResolvedValue(client);
const auth = authFor("main");
const monitor = await acquireSharedMatrixClient({
auth,
role: "monitor",
startClient: false,
});
const transient = await acquireSharedMatrixClient({
auth,
role: "transient",
startClient: false,
});
transient.abortSignal.addEventListener(
"abort",
() => {
callOrder.push("transient-cancel");
void transient.release();
},
{ once: true },
);
monitor.registerMonitorRetirement(createMonitorRetirement(callOrder));
await monitor.release({ mode: "persist" });
expect(transient.abortSignal.aborted).toBe(true);
expect(callOrder).toEqual([
"transient-cancel",
"quiesce",
"drain-quiesce",
"close-admission",
"detach-listeners",
"wait-tasks",
"monitor-cleanup",
"drain-final",
"persist",
]);
expect(client.stopWithoutPersist).not.toHaveBeenCalled();
});
it("lets a later waiter abort while shared startup continues for the owner", async () => {
const { mainClient, resolveStartup } = createPendingSharedStartup();
it("bounds non-cooperative transient drain and makes late release harmless", async () => {
vi.useFakeTimers();
const callOrder: string[] = [];
const client = createMockClient("main", callOrder);
createMatrixClientMock.mockResolvedValue(client);
const auth = authFor("main");
const monitor = await acquireSharedMatrixClient({
auth,
role: "monitor",
startClient: false,
});
const transient = await acquireSharedMatrixClient({
auth,
role: "transient",
startClient: false,
});
const ownerPromise = resolveSharedMatrixClient({ cfg: TEST_CFG, accountId: "main" });
monitor.registerMonitorRetirement(createMonitorRetirement(callOrder));
const retirement = monitor.release({ mode: "persist" });
const retirementError = retirement.then(
() => null,
(error: unknown) => error,
);
await vi.advanceTimersByTimeAsync(4_999);
expect(client.stopWithoutPersist).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
await expect(retirementError).resolves.toMatchObject({
message: "Matrix transient leases did not drain within 5000ms",
});
expect(transient.abortSignal.aborted).toBe(true);
expect(client.stopWithoutPersist).toHaveBeenCalledTimes(1);
expect(client.stopAndPersist).not.toHaveBeenCalled();
const firstLateRelease = transient.release({ mode: "persist" });
const secondLateRelease = transient.release({ mode: "persist" });
expect(secondLateRelease).toBe(firstLateRelease);
await firstLateRelease;
expect(client.stopWithoutPersist).toHaveBeenCalledTimes(1);
expect(client.stopAndPersist).not.toHaveBeenCalled();
});
it("quiesces and cleans up the monitor before waiting for an existing transient lease", async () => {
const callOrder: string[] = [];
const client = createMockClient("main", callOrder);
createMatrixClientMock.mockResolvedValue(client);
const auth = authFor("main");
const monitor = await acquireSharedMatrixClient({
auth,
role: "monitor",
startClient: false,
});
const transient = await acquireSharedMatrixClient({
auth,
role: "transient",
startClient: false,
});
monitor.registerMonitorRetirement(createMonitorRetirement(callOrder));
const monitorRelease = monitor.release({ mode: "persist" });
await vi.waitFor(() => {
expect(mainClient.start).toHaveBeenCalledTimes(1);
expect(callOrder).toContain("monitor-cleanup");
});
expect(callOrder).not.toContain("persist");
await transient.release({ mode: "stop" });
await monitorRelease;
expect(callOrder).toEqual([
"quiesce",
"drain-quiesce",
"close-admission",
"detach-listeners",
"wait-tasks",
"monitor-cleanup",
"drain-final",
"persist",
]);
});
it("keeps shared sync open until the final monitor lease is released", async () => {
const callOrder: string[] = [];
const client = createMockClient("main", callOrder);
createMatrixClientMock.mockResolvedValue(client);
const auth = authFor("main");
const first = await acquireSharedMatrixClient({
auth,
role: "monitor",
startClient: false,
});
const final = await acquireSharedMatrixClient({
auth,
role: "monitor",
startClient: false,
});
const firstRetirement = createMonitorRetirement(callOrder);
first.registerMonitorRetirement(firstRetirement);
final.registerMonitorRetirement(createMonitorRetirement(callOrder));
const firstRelease = first.release({ mode: "persist" });
expect(first.release({ mode: "discard" })).toBe(firstRelease);
await firstRelease;
expect(callOrder).toEqual([
"close-admission",
"detach-listeners",
"wait-tasks",
"monitor-cleanup",
]);
expect(client.quiesceSync).not.toHaveBeenCalled();
expect(client.stopAndPersist).not.toHaveBeenCalled();
expect(client.stopWithoutPersist).not.toHaveBeenCalled();
await final.release({ mode: "persist" });
expect(callOrder).toEqual([
"close-admission",
"detach-listeners",
"wait-tasks",
"monitor-cleanup",
"quiesce",
"drain-quiesce",
"close-admission",
"detach-listeners",
"wait-tasks",
"monitor-cleanup",
"drain-final",
"persist",
]);
expect(client.quiesceSync).toHaveBeenCalledTimes(1);
expect(client.stopAndPersist).toHaveBeenCalledTimes(1);
});
it("uses the same quiesce-before-persist path for a transient-only started generation", async () => {
const callOrder: string[] = [];
const client = createMockClient("main", callOrder);
createMatrixClientMock.mockResolvedValue(client);
const transient = await acquireSharedMatrixClient({
auth: authFor("main"),
role: "transient",
});
await transient.release({ mode: "persist" });
expect(callOrder).toEqual(["start", "quiesce", "drain-quiesce", "drain-final", "persist"]);
});
it("memoizes one release promise for duplicate release calls", async () => {
const client = createMockClient("main");
const persist = createDeferred();
client.stopAndPersist.mockReturnValue(persist.promise);
createMatrixClientMock.mockResolvedValue(client);
const lease = await acquireSharedMatrixClient({
auth: authFor("main"),
startClient: false,
});
const first = lease.release({ mode: "persist" });
const second = lease.release({ mode: "persist" });
expect(second).toBe(first);
persist.resolve();
await first;
await lease.release({ mode: "persist" });
expect(client.stopAndPersist).toHaveBeenCalledTimes(1);
});
it("waits abortably instead of admitting a new lease while a generation retires", async () => {
const firstClient = createMockClient("first");
const replacementClient = createMockClient("replacement");
const persist = createDeferred();
firstClient.stopAndPersist.mockReturnValue(persist.promise);
createMatrixClientMock
.mockResolvedValueOnce(firstClient)
.mockResolvedValueOnce(replacementClient);
const auth = authFor("main");
const lease = await acquireSharedMatrixClient({ auth, startClient: false });
const release = lease.release({ mode: "persist" });
await vi.waitFor(() => {
expect(firstClient.stopAndPersist).toHaveBeenCalledTimes(1);
});
const abortController = new AbortController();
const canceledWaiter = resolveSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
const blockedAcquire = acquireSharedMatrixClient({
auth,
abortSignal: abortController.signal,
});
abortController.abort();
await expectMatrixStartupAbort(blockedAcquire);
expect(createMatrixClientMock).toHaveBeenCalledTimes(1);
await expectMatrixStartupAbort(canceledWaiter);
resolveStartup();
await expect(ownerPromise).resolves.toBe(mainClient);
persist.resolve();
await release;
const replacement = await acquireSharedMatrixClient({ auth, startClient: false });
expect(replacement.client).toBe(replacementClient);
await replacement.release();
});
it("keeps the shared startup lock while an aborted waiter exits early", async () => {
const { mainClient, resolveStartup } = createPendingSharedStartup();
it("poisons a timed-out generation, discards final state, and never reopens it automatically", async () => {
const cause = new Error("Matrix classic sync did not reach STOPPED within 5000ms");
const callOrder: string[] = [];
const client = createMockClient("main", callOrder);
client.quiesceSync.mockImplementation(async () => {
callOrder.push("quiesce");
throw cause;
});
createMatrixClientMock.mockResolvedValue(client);
const auth = authFor("main");
const monitor = await acquireSharedMatrixClient({
auth,
role: "monitor",
startClient: false,
});
const transient = await acquireSharedMatrixClient({
auth,
role: "transient",
startClient: false,
});
const ownerPromise = resolveSharedMatrixClient({ cfg: TEST_CFG, accountId: "main" });
monitor.registerMonitorRetirement(createMonitorRetirement(callOrder));
const release = monitor.release({ mode: "persist" });
const releaseError = release.then(
() => null,
(error: unknown) => error,
);
await vi.waitFor(() => {
expect(mainClient.start).toHaveBeenCalledTimes(1);
expect(callOrder).toContain("monitor-cleanup");
});
await expect(acquireSharedMatrixClient({ auth })).rejects.toBe(cause);
await expect(transient.release()).rejects.toBe(cause);
expect(await releaseError).toBe(cause);
expect(client.stopWithoutPersist).toHaveBeenCalledTimes(1);
expect(client.stopAndPersist).not.toHaveBeenCalled();
await expect(acquireSharedMatrixClient({ auth })).rejects.toBe(cause);
expect(createMatrixClientMock).toHaveBeenCalledTimes(1);
});
it("preserves an earlier stop requirement when the final lease requests discard", async () => {
const cause = new Error("best-effort persistence failed");
const persist = createDeferred();
const firstClient = createMockClient("first");
const replacementClient = createMockClient("replacement");
firstClient.stopAndPersist.mockReturnValue(persist.promise);
createMatrixClientMock
.mockResolvedValueOnce(firstClient)
.mockResolvedValueOnce(replacementClient);
const auth = authFor("main");
const first = await acquireSharedMatrixClient({ auth, startClient: false });
const final = await acquireSharedMatrixClient({ auth, startClient: false });
await first.release({ mode: "stop" });
expect(firstClient.stopAndPersist).not.toHaveBeenCalled();
const release = final.release({ mode: "discard" });
await vi.waitFor(() => {
expect(firstClient.stopAndPersist).toHaveBeenCalledTimes(1);
});
const replacementPromise = acquireSharedMatrixClient({ auth, startClient: false });
expect(createMatrixClientMock).toHaveBeenCalledTimes(1);
persist.reject(cause);
await release;
expect(firstClient.stopWithoutPersist).toHaveBeenCalledTimes(1);
const replacement = await replacementPromise;
expect(replacement.client).toBe(replacementClient);
await replacement.release({ mode: "discard" });
});
it("discards without attempting persistence", async () => {
const client = createMockClient("main");
createMatrixClientMock.mockResolvedValue(client);
const lease = await acquireSharedMatrixClient({
auth: authFor("main"),
startClient: false,
});
await lease.release({ mode: "discard" });
expect(client.stopAndPersist).not.toHaveBeenCalled();
expect(client.stopWithoutPersist).toHaveBeenCalledTimes(1);
});
it("preserves and propagates an earlier persist requirement when final lease discards", async () => {
const cause = new Error("crypto persist failed");
const firstClient = createMockClient("first");
const replacementClient = createMockClient("replacement");
firstClient.stopAndPersist.mockRejectedValue(cause);
createMatrixClientMock
.mockResolvedValueOnce(firstClient)
.mockResolvedValueOnce(replacementClient);
const auth = authFor("main");
const first = await acquireSharedMatrixClient({ auth, startClient: false });
const final = await acquireSharedMatrixClient({ auth, startClient: false });
await first.release({ mode: "persist" });
expect(firstClient.stopAndPersist).not.toHaveBeenCalled();
await expect(final.release({ mode: "discard" })).rejects.toBe(cause);
expect(firstClient.stopWithoutPersist).not.toHaveBeenCalled();
const replacement = await acquireSharedMatrixClient({ auth, startClient: false });
expect(replacement.client).toBe(replacementClient);
await replacement.release();
});
it("discards and replaces a generation when the final decryption drain fails", async () => {
const cause = new Error("final decryption drain timed out");
const firstClient = createMockClient("first");
const replacementClient = createMockClient("replacement");
firstClient.drainPendingDecryptions
.mockResolvedValueOnce(undefined)
.mockRejectedValueOnce(cause);
createMatrixClientMock
.mockResolvedValueOnce(firstClient)
.mockResolvedValueOnce(replacementClient);
const auth = authFor("main");
const lease = await acquireSharedMatrixClient({ auth, startClient: false });
await expect(lease.release({ mode: "persist" })).rejects.toBe(cause);
expect(firstClient.stopWithoutPersist).toHaveBeenCalledTimes(1);
expect(firstClient.stopAndPersist).not.toHaveBeenCalled();
await expect(stopSharedClientForAccount(auth)).resolves.toBeUndefined();
const replacement = await acquireSharedMatrixClient({ auth, startClient: false });
expect(replacement.client).toBe(replacementClient);
await replacement.release({ mode: "discard" });
});
it("aborts a first starter and waiter during forced retirement without late reuse", async () => {
const start = createDeferred();
const firstClient = createMockClient("first");
const replacementClient = createMockClient("replacement");
let startupSignal: AbortSignal | undefined;
firstClient.start.mockImplementation(({ abortSignal } = {}) => {
startupSignal = abortSignal;
return start.promise;
});
createMatrixClientMock
.mockResolvedValueOnce(firstClient)
.mockResolvedValueOnce(replacementClient);
const auth = authFor("main");
const owner = await acquireSharedMatrixClient({ auth, startClient: false });
const waiter = await acquireSharedMatrixClient({ auth, startClient: false });
const callerAbort = new AbortController();
const ownerStart = owner.start(callerAbort.signal);
await vi.waitFor(() => {
expect(firstClient.start).toHaveBeenCalledTimes(1);
});
const waiterStart = waiter.start();
const ownerAbort = expectMatrixStartupAbort(ownerStart);
const waiterAbort = expectMatrixStartupAbort(waiterStart);
await Promise.all([stopSharedClientForAccount(auth), ownerAbort, waiterAbort]);
expect(callerAbort.signal.aborted).toBe(false);
expect(startupSignal?.aborted).toBe(true);
expect(owner.abortSignal.aborted).toBe(true);
expect(waiter.abortSignal.aborted).toBe(true);
start.resolve();
await Promise.resolve();
const replacement = await acquireSharedMatrixClient({ auth, startClient: false });
expect(replacement.client).toBe(replacementClient);
await replacement.release({ mode: "discard" });
});
it("does not let one aborted startup waiter remove another lease", async () => {
const client = createMockClient("main");
const start = createDeferred();
client.start.mockReturnValue(start.promise);
createMatrixClientMock.mockResolvedValue(client);
const auth = authFor("main");
const owner = await acquireSharedMatrixClient({
auth,
startClient: false,
});
const ownerStart = owner.start();
const abortController = new AbortController();
const abortedWaiter = resolveSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
const waiter = acquireSharedMatrixClient({
auth,
abortSignal: abortController.signal,
});
abortController.abort();
await expectMatrixStartupAbort(abortedWaiter);
await expectMatrixStartupAbort(waiter);
const followerPromise = resolveSharedMatrixClient({ cfg: TEST_CFG, accountId: "main" });
expect(mainClient.start).toHaveBeenCalledTimes(1);
resolveStartup();
await expect(ownerPromise).resolves.toBe(mainClient);
await expect(followerPromise).resolves.toBe(mainClient);
expect(mainClient.start).toHaveBeenCalledTimes(1);
});
it("recreates the shared client when dispatcherPolicy changes", async () => {
const firstAuth = {
...authFor("main"),
dispatcherPolicy: {
mode: "explicit-proxy" as const,
proxyUrl: "http://127.0.0.1:7890",
},
};
const secondAuth = {
...authFor("main"),
dispatcherPolicy: {
mode: "explicit-proxy" as const,
proxyUrl: "http://127.0.0.1:7891",
},
};
const firstClient = createMockClient("main-first");
const secondClient = createMockClient("main-second");
resolveMatrixAuthMock.mockResolvedValueOnce(firstAuth).mockResolvedValueOnce(secondAuth);
createMatrixClientMock.mockResolvedValueOnce(firstClient).mockResolvedValueOnce(secondClient);
const first = await resolveSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
startClient: false,
});
const second = await resolveSharedMatrixClient({
cfg: TEST_CFG,
accountId: "main",
startClient: false,
});
expect(first).toBe(firstClient);
expect(second).toBe(secondClient);
expect(createMatrixClientMock).toHaveBeenCalledTimes(2);
start.resolve();
await ownerStart;
await owner.release();
expect(client.start).toHaveBeenCalledTimes(1);
});
});
+407 -193
View File
@@ -1,5 +1,6 @@
// Matrix plugin module implements shared behavior.
import { normalizeOptionalAccountId } from "openclaw/plugin-sdk/account-id";
import { createDeferred } from "openclaw/plugin-sdk/extension-shared";
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
import type { CoreConfig } from "../../types.js";
import type { MatrixClient } from "../sdk.js";
@@ -13,14 +14,62 @@ const loadMatrixCreateClientDeps = createLazyRuntimeModule(() =>
createMatrixClient: runtime.createMatrixClient,
})),
);
const MATRIX_TRANSIENT_LEASE_DRAIN_TIMEOUT_MS = 5_000;
export type MatrixClientLeaseRole = "monitor" | "transient";
export type MatrixClientReleaseMode = "stop" | "persist" | "discard";
export type MatrixMonitorRetirement = {
closeTaskAdmission: () => void;
detachListeners: () => void;
waitForTasks: () => Promise<void>;
cleanup: () => Promise<void> | void;
};
export type SharedMatrixClientLease = {
abortSignal: AbortSignal;
client: MatrixClient;
role: MatrixClientLeaseRole;
registerMonitorRetirement: (retirement: MatrixMonitorRetirement) => void;
start: (abortSignal?: AbortSignal) => Promise<void>;
release: (params?: { mode?: MatrixClientReleaseMode }) => Promise<void>;
};
type SharedMatrixClientPhase = "open" | "quiescing" | "closing";
type SharedMatrixClientLeaseState = {
abortController: AbortController;
monitorRetirement: MatrixMonitorRetirement | null;
monitorRetirementPromise: Promise<void> | null;
role: MatrixClientLeaseRole;
releasePromise: Promise<void> | null;
};
type SharedMatrixClientState = {
auth: MatrixAuth;
client: MatrixClient;
key: string;
started: boolean;
cryptoReady: boolean;
startPromise: Promise<void> | null;
leases: number;
phase: SharedMatrixClientPhase;
leases: Set<SharedMatrixClientLeaseState>;
monitorRetirementPromises: Set<Promise<void>>;
noLeases: { promise: Promise<void>; resolve: () => void };
retirementPromise: Promise<void> | null;
poisonError: Error | null;
releaseMode: MatrixClientReleaseMode;
};
type SharedMatrixClientParams = {
cfg?: CoreConfig;
env?: NodeJS.ProcessEnv;
timeoutMs?: number;
auth?: MatrixAuth;
startClient?: boolean;
accountId?: string | null;
abortSignal?: AbortSignal;
role?: MatrixClientLeaseRole;
};
const sharedClientStates = new Map<string, SharedMatrixClientState>();
@@ -62,250 +111,415 @@ async function createSharedMatrixClient(params: {
dispatcherPolicy: params.auth.dispatcherPolicy,
});
return {
auth: params.auth,
client,
key: buildSharedClientKey(params.auth),
started: false,
cryptoReady: false,
startPromise: null,
leases: 0,
phase: "open",
leases: new Set(),
monitorRetirementPromises: new Set(),
noLeases: createDeferred<void>(),
retirementPromise: null,
poisonError: null,
releaseMode: "discard",
};
}
function findSharedClientStateByInstance(client: MatrixClient): SharedMatrixClientState | null {
for (const state of sharedClientStates.values()) {
if (state.client === client) {
return state;
}
}
return null;
}
function deleteSharedClientState(state: SharedMatrixClientState): void {
sharedClientStates.delete(state.key);
if (sharedClientStates.get(state.key) === state) {
sharedClientStates.delete(state.key);
}
sharedClientPromises.delete(state.key);
}
async function ensureSharedClientStarted(params: {
state: SharedMatrixClientState;
encryption?: boolean;
abortSignal?: AbortSignal;
}): Promise<void> {
const waitForStart = async (startPromise: Promise<void>) => {
await awaitMatrixStartupWithAbort(startPromise, params.abortSignal);
};
if (params.state.started) {
async function ensureSharedClientStarted(
state: SharedMatrixClientState,
abortSignal?: AbortSignal,
): Promise<void> {
if (state.started) {
return;
}
if (params.state.startPromise) {
await waitForStart(params.state.startPromise);
if (state.startPromise) {
await awaitMatrixStartupWithAbort(state.startPromise, abortSignal);
return;
}
const startPromise = (async () => {
const client = params.state.client;
// Initialize crypto if enabled
if (params.encryption && !params.state.cryptoReady) {
if (state.auth.encryption && !state.cryptoReady) {
try {
const joinedRooms = await client.getJoinedRooms();
if (client.crypto) {
await client.crypto.prepare(joinedRooms);
params.state.cryptoReady = true;
const joinedRooms = await state.client.getJoinedRooms();
if (state.client.crypto) {
await state.client.crypto.prepare(joinedRooms);
state.cryptoReady = true;
}
} catch (err) {
LogService.warn("MatrixClientLite", "Failed to prepare crypto:", err);
}
}
await client.start({ abortSignal: params.abortSignal });
params.state.started = true;
await awaitMatrixStartupWithAbort(state.client.start({ abortSignal }), abortSignal);
state.started = true;
})();
// Keep the shared startup lock until the underlying start fully settles, even
// if one waiter aborts early while another caller still owns the startup.
const guardedStart = startPromise.finally(() => {
if (params.state.startPromise === guardedStart) {
params.state.startPromise = null;
if (state.startPromise === guardedStart) {
state.startPromise = null;
}
});
params.state.startPromise = guardedStart;
await waitForStart(guardedStart);
state.startPromise = guardedStart;
await awaitMatrixStartupWithAbort(guardedStart, abortSignal);
}
async function resolveSharedMatrixClientState(
params: {
cfg?: CoreConfig;
env?: NodeJS.ProcessEnv;
timeoutMs?: number;
auth?: MatrixAuth;
startClient?: boolean;
accountId?: string | null;
abortSignal?: AbortSignal;
} = {},
): Promise<SharedMatrixClientState> {
async function resolveSharedMatrixAuth(params: SharedMatrixClientParams): Promise<MatrixAuth> {
const requestedAccountId = normalizeOptionalAccountId(params.accountId);
if (params.auth && requestedAccountId && requestedAccountId !== params.auth.accountId) {
throw new Error(
`Matrix shared client account mismatch: requested ${requestedAccountId}, auth resolved ${params.auth.accountId}`,
);
}
const authContext = (() => {
if (params.auth) {
return null;
}
if (!params.cfg) {
throw new Error(
"Matrix shared client requires a resolved runtime config. Load and resolve config at the command or gateway boundary, then pass cfg through the runtime path.",
);
}
return resolveMatrixAuthContext({
cfg: params.cfg,
env: params.env,
accountId: params.accountId,
});
})();
const auth =
params.auth ??
(await resolveMatrixAuth({
cfg: authContext?.cfg ?? params.cfg,
env: authContext?.env ?? params.env,
accountId: authContext?.accountId,
}));
const key = buildSharedClientKey(auth);
const shouldStart = params.startClient !== false;
const existingState = sharedClientStates.get(key);
if (existingState) {
if (shouldStart) {
await ensureSharedClientStarted({
state: existingState,
encryption: auth.encryption,
abortSignal: params.abortSignal,
});
}
return existingState;
if (params.auth) {
return params.auth;
}
const existingPromise = sharedClientPromises.get(key);
if (existingPromise) {
const pending = await existingPromise;
if (shouldStart) {
await ensureSharedClientStarted({
state: pending,
encryption: auth.encryption,
abortSignal: params.abortSignal,
});
}
return pending;
if (!params.cfg) {
throw new Error(
"Matrix shared client requires a resolved runtime config. Load and resolve config at the command or gateway boundary, then pass cfg through the runtime path.",
);
}
const creationPromise = createSharedMatrixClient({
auth,
timeoutMs: params.timeoutMs,
const authContext = resolveMatrixAuthContext({
cfg: params.cfg,
env: params.env,
accountId: params.accountId,
});
sharedClientPromises.set(key, creationPromise);
return await resolveMatrixAuth({
cfg: authContext.cfg,
env: authContext.env,
accountId: authContext.accountId,
});
}
try {
const created = await creationPromise;
sharedClientStates.set(key, created);
if (shouldStart) {
await ensureSharedClientStarted({
state: created,
encryption: auth.encryption,
abortSignal: params.abortSignal,
});
async function resolveOpenSharedMatrixClientState(
params: SharedMatrixClientParams,
): Promise<SharedMatrixClientState> {
const auth = await resolveSharedMatrixAuth(params);
const key = buildSharedClientKey(auth);
while (true) {
const existing = sharedClientStates.get(key);
if (existing?.poisonError) {
throw existing.poisonError;
}
if (existing?.phase === "open") {
return existing;
}
if (existing?.retirementPromise) {
await awaitMatrixStartupWithAbort(existing.retirementPromise, params.abortSignal);
continue;
}
const pending = sharedClientPromises.get(key);
if (pending) {
await awaitMatrixStartupWithAbort(pending, params.abortSignal);
continue;
}
const creationPromise = createSharedMatrixClient({
auth,
timeoutMs: params.timeoutMs,
});
sharedClientPromises.set(key, creationPromise);
try {
const created = await creationPromise;
sharedClientStates.set(key, created);
return created;
} finally {
sharedClientPromises.delete(key);
}
return created;
} finally {
sharedClientPromises.delete(key);
}
}
export async function resolveSharedMatrixClient(
params: {
cfg?: CoreConfig;
env?: NodeJS.ProcessEnv;
timeoutMs?: number;
auth?: MatrixAuth;
startClient?: boolean;
accountId?: string | null;
abortSignal?: AbortSignal;
} = {},
): Promise<MatrixClient> {
const state = await resolveSharedMatrixClientState(params);
return state.client;
async function runMonitorRetirement(
retirement: MatrixMonitorRetirement | undefined,
): Promise<void> {
if (!retirement) {
return;
}
retirement.closeTaskAdmission();
retirement.detachListeners();
await retirement.waitForTasks();
await retirement.cleanup();
}
function retireMonitorLease(
state: SharedMatrixClientState,
lease: SharedMatrixClientLeaseState,
): Promise<void> {
if (lease.monitorRetirementPromise) {
return lease.monitorRetirementPromise;
}
lease.monitorRetirementPromise = runMonitorRetirement(lease.monitorRetirement ?? undefined);
state.monitorRetirementPromises.add(lease.monitorRetirementPromise);
return lease.monitorRetirementPromise;
}
async function retireMonitorLeases(
state: SharedMatrixClientState,
leases: SharedMatrixClientLeaseState[],
): Promise<void> {
for (const lease of leases) {
void retireMonitorLease(state, lease);
}
const results = await Promise.allSettled(state.monitorRetirementPromises);
const failure = results.find((result) => result.status === "rejected");
if (failure?.status === "rejected") {
throw failure.reason;
}
}
function toRetirementError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
function mergeReleaseMode(
current: MatrixClientReleaseMode,
requested: MatrixClientReleaseMode,
): MatrixClientReleaseMode {
// Release requirements belong to the generation; one lease cannot weaken another's durability.
if (current === "persist" || requested === "persist") {
return "persist";
}
if (current === "stop" || requested === "stop") {
return "stop";
}
return "discard";
}
function abortTransientLeases(state: SharedMatrixClientState): void {
for (const lease of state.leases) {
if (lease.role === "transient") {
lease.abortController.abort();
}
}
}
function forceReleaseLeases(
state: SharedMatrixClientState,
releasePromise = Promise.resolve(),
): void {
for (const lease of state.leases) {
lease.abortController.abort();
lease.releasePromise ??= releasePromise;
}
state.leases.clear();
state.noLeases.resolve();
}
async function waitForLeaseDrain(state: SharedMatrixClientState): Promise<void> {
if (state.leases.size === 0) {
return;
}
let deadline: NodeJS.Timeout | undefined;
try {
await Promise.race([
state.noLeases.promise,
new Promise<never>((_, reject) => {
deadline = setTimeout(() => {
reject(
new Error(
`Matrix transient leases did not drain within ${MATRIX_TRANSIENT_LEASE_DRAIN_TIMEOUT_MS}ms`,
),
);
}, MATRIX_TRANSIENT_LEASE_DRAIN_TIMEOUT_MS);
deadline.unref?.();
}),
]);
} finally {
if (deadline) {
clearTimeout(deadline);
}
}
}
function beginGenerationRetirement(params: {
state: SharedMatrixClientState;
monitorLeases?: SharedMatrixClientLeaseState[];
}): Promise<void> {
const { state } = params;
if (state.retirementPromise) {
return state.retirementPromise;
}
state.phase = "quiescing";
state.retirementPromise = Promise.resolve().then(async () => {
try {
await state.client.quiesceSync();
state.started = false;
await state.client.drainPendingDecryptions("matrix monitor sync quiesce");
} catch (error) {
state.poisonError = toRetirementError(error);
}
try {
await retireMonitorLeases(state, params.monitorLeases ?? []);
} catch (error) {
state.poisonError ??= toRetirementError(error);
}
state.phase = "closing";
try {
await waitForLeaseDrain(state);
} catch (error) {
state.poisonError ??= toRetirementError(error);
forceReleaseLeases(state);
}
if (state.poisonError) {
await state.client
.drainPendingDecryptions("matrix poisoned client shutdown")
.catch(() => undefined);
state.client.stopWithoutPersist();
throw state.poisonError;
}
try {
await state.client.drainPendingDecryptions("matrix shared client final shutdown");
} catch (error) {
state.poisonError = toRetirementError(error);
try {
state.client.stopWithoutPersist();
} finally {
deleteSharedClientState(state);
}
throw state.poisonError;
}
try {
if (state.releaseMode === "persist") {
await state.client.stopAndPersist();
} else if (state.releaseMode === "discard") {
state.client.stopWithoutPersist();
} else {
await state.client.stopAndPersist().catch(() => state.client.stopWithoutPersist());
}
} finally {
deleteSharedClientState(state);
}
});
abortTransientLeases(state);
return state.retirementPromise;
}
function createSharedMatrixClientLease(
state: SharedMatrixClientState,
role: MatrixClientLeaseRole,
): SharedMatrixClientLease {
const leaseState: SharedMatrixClientLeaseState = {
abortController: new AbortController(),
monitorRetirement: null,
monitorRetirementPromise: null,
role,
releasePromise: null,
};
state.leases.add(leaseState);
return {
abortSignal: leaseState.abortController.signal,
client: state.client,
role,
registerMonitorRetirement: (retirement) => {
if (role !== "monitor") {
throw new Error("Matrix transient leases cannot register monitor retirement");
}
if (leaseState.releasePromise || state.phase !== "open") {
throw new Error("Matrix monitor lease is already retiring");
}
if (leaseState.monitorRetirement && leaseState.monitorRetirement !== retirement) {
throw new Error("Matrix monitor retirement is already registered");
}
leaseState.monitorRetirement = retirement;
},
start: async (abortSignal) => {
if (leaseState.releasePromise) {
throw new Error("Matrix client lease has already been released");
}
if (state.phase !== "open") {
throw new Error("Matrix client generation is retiring");
}
const startupSignal = abortSignal
? AbortSignal.any([abortSignal, leaseState.abortController.signal])
: leaseState.abortController.signal;
await ensureSharedClientStarted(state, startupSignal);
},
release: (releaseParams = {}) => {
if (leaseState.releasePromise) {
return leaseState.releasePromise;
}
state.releaseMode = mergeReleaseMode(state.releaseMode, releaseParams.mode ?? "stop");
state.leases.delete(leaseState);
if (state.leases.size === 0) {
state.noLeases.resolve();
}
const finalMonitor =
role === "monitor" && !Array.from(state.leases).some((lease) => lease.role === "monitor");
if (role === "monitor" && !finalMonitor) {
leaseState.releasePromise = retireMonitorLease(state, leaseState);
return leaseState.releasePromise;
}
const shouldRetire = finalMonitor || state.leases.size === 0;
if (!shouldRetire) {
leaseState.releasePromise = Promise.resolve();
return leaseState.releasePromise;
}
leaseState.releasePromise = beginGenerationRetirement({
state,
monitorLeases: role === "monitor" ? [leaseState] : undefined,
});
return leaseState.releasePromise;
},
};
}
export async function acquireSharedMatrixClient(
params: {
cfg?: CoreConfig;
env?: NodeJS.ProcessEnv;
timeoutMs?: number;
auth?: MatrixAuth;
startClient?: boolean;
accountId?: string | null;
abortSignal?: AbortSignal;
} = {},
): Promise<MatrixClient> {
const state = await resolveSharedMatrixClientState(params);
state.leases += 1;
return state.client;
}
export function stopSharedClient(): void {
for (const state of sharedClientStates.values()) {
state.client.stop();
params: SharedMatrixClientParams = {},
): Promise<SharedMatrixClientLease> {
const state = await resolveOpenSharedMatrixClientState(params);
const lease = createSharedMatrixClientLease(state, params.role ?? "transient");
if (params.startClient !== false) {
try {
await lease.start(params.abortSignal);
} catch (error) {
await lease.release({ mode: "stop" }).catch(() => undefined);
throw error;
}
}
sharedClientStates.clear();
sharedClientPromises.clear();
return lease;
}
export function stopSharedClientForAccount(auth: MatrixAuth): void {
const key = buildSharedClientKey(auth);
const state = sharedClientStates.get(key);
async function forceRetireState(state: SharedMatrixClientState): Promise<void> {
state.releaseMode = mergeReleaseMode(state.releaseMode, "stop");
const retirementPromise = beginGenerationRetirement({
state,
monitorLeases: Array.from(state.leases).filter((lease) => lease.role === "monitor"),
});
forceReleaseLeases(state, retirementPromise);
if (state.poisonError) {
await retirementPromise.catch(() => undefined);
deleteSharedClientState(state);
return;
}
await retirementPromise.catch((error: unknown) => {
if (!state.poisonError) {
throw error;
}
});
if (state.poisonError) {
deleteSharedClientState(state);
}
}
export async function stopSharedClientForAccount(auth: MatrixAuth): Promise<void> {
const state = sharedClientStates.get(buildSharedClientKey(auth));
if (!state) {
return;
}
state.client.stop();
deleteSharedClientState(state);
}
export function removeSharedClientInstance(client: MatrixClient): boolean {
const state = findSharedClientStateByInstance(client);
if (!state) {
return false;
}
deleteSharedClientState(state);
return true;
}
export function stopSharedClientInstance(client: MatrixClient): void {
if (!removeSharedClientInstance(client)) {
return;
}
client.stop();
}
export async function releaseSharedClientInstance(
client: MatrixClient,
mode: "stop" | "persist" | "discard" = "stop",
): Promise<boolean> {
const state = findSharedClientStateByInstance(client);
if (!state) {
return false;
}
state.leases = Math.max(0, state.leases - 1);
if (state.leases > 0) {
return false;
}
deleteSharedClientState(state);
if (mode === "persist") {
await client.stopAndPersist();
} else if (mode === "discard") {
client.stopWithoutPersist();
} else {
client.stop();
}
return true;
await forceRetireState(state);
}
@@ -22,6 +22,12 @@ function createClientStub() {
}
return client;
}),
off: vi.fn((eventName: string, listener: unknown) => {
if (eventName === "room.invite" && inviteHandler === listener) {
inviteHandler = null;
}
return client;
}),
joinRoom: vi.fn(async () => {}),
resolveRoom: vi.fn(async () => null),
} as unknown as import("../sdk.js").MatrixClient;
@@ -30,6 +36,7 @@ function createClientStub() {
client,
getInviteHandler: () => inviteHandler,
joinRoom: (client as unknown as { joinRoom: ReturnType<typeof vi.fn> }).joinRoom,
off: (client as unknown as { off: ReturnType<typeof vi.fn> }).off,
resolveRoom: (client as unknown as { resolveRoom: ReturnType<typeof vi.fn> }).resolveRoom,
};
}
@@ -41,6 +48,9 @@ function registerAutoJoinHarness(params: {
error?: ReturnType<typeof vi.fn>;
}) {
const harness = createClientStub();
const runDetachedTask = vi.fn((_label: string, task: () => Promise<void>) =>
Promise.resolve().then(task),
);
if (params.resolveRoomValues) {
for (const value of params.resolveRoomValues) {
harness.resolveRoom.mockResolvedValueOnce(value);
@@ -49,16 +59,17 @@ function registerAutoJoinHarness(params: {
harness.resolveRoom.mockResolvedValue(params.resolveRoomValue);
}
registerMatrixAutoJoin({
const dispose = registerMatrixAutoJoin({
client: harness.client,
accountConfig: params.accountConfig ?? {},
runtime: {
log: vi.fn(),
error: params.error ?? vi.fn(),
} as unknown as RuntimeEnv,
runDetachedTask,
});
return harness;
return { ...harness, dispose, runDetachedTask };
}
async function triggerInvite(
@@ -213,4 +224,23 @@ describe("registerMatrixAutoJoin", () => {
await expect(triggerInvite(getInviteHandler, {})).resolves.toBeUndefined();
});
it("removes the exact invite listener on disposal", async () => {
const { dispose, getInviteHandler, joinRoom, off, runDetachedTask } = registerAutoJoinHarness({
accountConfig: {
autoJoin: "always",
},
});
const listener = getInviteHandler();
if (!listener) {
throw new Error("expected Matrix invite handler");
}
dispose();
expect(off).toHaveBeenCalledWith("room.invite", listener);
expect(getInviteHandler()).toBeNull();
expect(runDetachedTask).not.toHaveBeenCalled();
expect(joinRoom).not.toHaveBeenCalled();
});
});
@@ -9,7 +9,8 @@ export function registerMatrixAutoJoin(params: {
client: MatrixClient;
accountConfig: Pick<MatrixConfig, "autoJoin" | "autoJoinAllowlist">;
runtime: RuntimeEnv;
}) {
runDetachedTask: (label: string, task: () => Promise<void>) => Promise<void>;
}): () => void {
const { client, accountConfig, runtime } = params;
const core = getMatrixRuntime();
const logVerbose = (message: string) => {
@@ -26,7 +27,7 @@ export function registerMatrixAutoJoin(params: {
const resolvedAliasRoomIds = new Map<string, string>();
if (autoJoin === "off") {
return;
return () => {};
}
if (autoJoin === "always") {
@@ -59,19 +60,9 @@ export function registerMatrixAutoJoin(params: {
);
return resolved.filter((roomId): roomId is string => Boolean(roomId));
};
const runInviteTask = (roomId: string, task: () => Promise<void>) => {
void Promise.resolve()
.then(task)
.catch((err: unknown) => {
runtime.error?.(
`matrix: auto-join invite handler failed for room ${roomId}: ${String(err)}`,
);
});
};
// Handle invites directly so both "always" and "allowlist" modes share the same path.
client.on("room.invite", (roomId: string, _inviteEvent: unknown) => {
runInviteTask(roomId, async () => {
const onInvite = (roomId: string, _inviteEvent: unknown) => {
void params.runDetachedTask(`auto-join invite handler room=${roomId}`, async () => {
if (autoJoin === "allowlist") {
const allowedAliasRoomIds = await resolveAllowedAliasRoomIds();
const allowed =
@@ -92,5 +83,9 @@ export function registerMatrixAutoJoin(params: {
runtime.error?.(`matrix: failed to join room ${roomId}: ${String(err)}`);
}
});
});
};
client.on("room.invite", onInvite);
return () => {
client.off("room.invite", onInvite);
};
}
@@ -146,6 +146,12 @@ function createHarness(params?: {
listeners.set(eventName, listener);
return client;
}),
off: vi.fn((eventName: string, listener: (...args: unknown[]) => void) => {
if (listeners.get(eventName) === listener) {
listeners.delete(eventName);
}
return client;
}),
sendMessage,
getUserId: vi.fn(async () => {
if (params?.selfUserIdError) {
@@ -186,7 +192,7 @@ function createHarness(params?: {
const dmPolicy = params?.dmPolicy ?? "open";
const allowFrom = params?.allowFrom ?? (dmPolicy === "open" ? ["*"] : []);
registerMatrixMonitorEvents({
const dispose = registerMatrixMonitorEvents({
cfg: params?.cfg ?? { channels: { matrix: {} } },
client,
auth: {
@@ -222,6 +228,7 @@ function createHarness(params?: {
}
return {
dispose,
onRoomMessage,
sendMessage,
invalidateRoom,
@@ -247,10 +254,29 @@ function createHarness(params?: {
| undefined,
roomInviteListener: listeners.get("room.invite") as RoomEventListener | undefined,
roomJoinListener: listeners.get("room.join") as RoomEventListener | undefined,
listenerCount: () => listeners.size,
off: (client as unknown as { off: ReturnType<typeof vi.fn> }).off,
on: (client as unknown as { on: ReturnType<typeof vi.fn> }).on,
};
}
describe("registerMatrixMonitorEvents verification routing", () => {
it("removes every exact monitor listener on disposal", () => {
const { dispose, listenerCount, off, on } = createHarness();
const registeredListeners = on.mock.calls.map(
([eventName, listener]) => [eventName, listener] as const,
);
expect(listenerCount()).toBe(8);
dispose();
expect(listenerCount()).toBe(0);
expect(off).toHaveBeenCalledTimes(8);
for (const [eventName, listener] of registeredListeners) {
expect(off).toHaveBeenCalledWith(eventName, listener);
}
});
it("does not repost historical verification completions during startup catch-up", async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-03-14T13:10:00.000Z"));
+38 -17
View File
@@ -5,6 +5,7 @@ import type { CoreConfig } from "../../types.js";
import type { MatrixAuth } from "../client.js";
import { formatMatrixEncryptedEventDisabledWarning } from "../encryption-guidance.js";
import type { MatrixClient } from "../sdk.js";
import type { MatrixVerificationSummary } from "../sdk/verification-manager.js";
import type { MatrixRawEvent } from "./types.js";
import { EventType } from "./types.js";
import { createMatrixVerificationEventRouter } from "./verification-events.js";
@@ -190,7 +191,7 @@ export function registerMatrixMonitorEvents(params: {
onRoomMessage: (roomId: string, event: MatrixRawEvent) => void | Promise<void>;
runDetachedTask?: (label: string, task: () => Promise<void>) => Promise<void>;
sasNoticeRetryDelayMs?: number;
}): void {
}): () => void {
const {
cfg,
client,
@@ -238,7 +239,7 @@ export function registerMatrixMonitorEvents(params: {
});
};
client.on("room.message", (roomId: string, event: MatrixRawEvent) => {
const onRoomMessageEvent = (roomId: string, event: MatrixRawEvent) => {
if (routeVerificationEvent(roomId, event)) {
return;
}
@@ -248,15 +249,15 @@ export function registerMatrixMonitorEvents(params: {
await onRoomMessage(roomId, event);
},
);
});
};
client.on("room.encrypted_event", (roomId: string, event: MatrixRawEvent) => {
const onEncryptedEvent = (roomId: string, event: MatrixRawEvent) => {
const eventId = event?.event_id ?? "unknown";
const eventType = event?.type ?? "unknown";
logVerboseMessage(`matrix: encrypted event room=${roomId} type=${eventType} id=${eventId}`);
});
};
client.on("room.decrypted_event", (roomId: string, event: MatrixRawEvent) => {
const onDecryptedEvent = (roomId: string, event: MatrixRawEvent) => {
const eventId = event?.event_id ?? "unknown";
const eventType = event?.type ?? "unknown";
logVerboseMessage(`matrix: decrypted event room=${roomId} type=${eventType} id=${eventId}`);
@@ -272,9 +273,9 @@ export function registerMatrixMonitorEvents(params: {
await onRoomMessage(roomId, event);
},
);
});
};
client.on("room.failed_decryption", (roomId: string, event: MatrixRawEvent, error: Error) => {
const onFailedDecryption = (roomId: string, event: MatrixRawEvent, error: Error) => {
void runMonitorTask(
`failed decryption handler room=${roomId} id=${event.event_id ?? "unknown"}`,
async () => {
@@ -330,15 +331,15 @@ export function registerMatrixMonitorEvents(params: {
);
},
);
});
};
client.on("verification.summary", (summary) => {
const onVerificationSummary = (summary: MatrixVerificationSummary) => {
void runMonitorTask("verification summary handler", async () => {
await routeVerificationSummary(summary);
});
});
};
client.on("room.invite", (roomId: string, event: MatrixRawEvent) => {
const onInvite = (roomId: string, event: MatrixRawEvent) => {
directTracker?.invalidateRoom(roomId);
const eventId = event?.event_id ?? "unknown";
const sender = event?.sender ?? "unknown";
@@ -353,15 +354,15 @@ export function registerMatrixMonitorEvents(params: {
logVerboseMessage(
`matrix: invite room=${roomId} sender=${sender} direct=${String(isDirect)} id=${eventId}`,
);
});
};
client.on("room.join", (roomId: string, event: MatrixRawEvent) => {
const onJoin = (roomId: string, event: MatrixRawEvent) => {
directTracker?.invalidateRoom(roomId);
const eventId = event?.event_id ?? "unknown";
logVerboseMessage(`matrix: join room=${roomId} id=${eventId}`);
});
};
client.on("room.event", (roomId: string, event: MatrixRawEvent) => {
const onRoomEvent = (roomId: string, event: MatrixRawEvent) => {
const eventType = event?.type ?? "unknown";
if (eventType === EventType.RoomMessageEncrypted) {
logVerboseMessage(
@@ -406,5 +407,25 @@ export function registerMatrixMonitorEvents(params: {
}
routeVerificationEvent(roomId, event);
});
};
client.on("room.message", onRoomMessageEvent);
client.on("room.encrypted_event", onEncryptedEvent);
client.on("room.decrypted_event", onDecryptedEvent);
client.on("room.failed_decryption", onFailedDecryption);
client.on("verification.summary", onVerificationSummary);
client.on("room.invite", onInvite);
client.on("room.join", onJoin);
client.on("room.event", onRoomEvent);
return () => {
client.off("room.message", onRoomMessageEvent);
client.off("room.encrypted_event", onEncryptedEvent);
client.off("room.decrypted_event", onDecryptedEvent);
client.off("room.failed_decryption", onFailedDecryption);
client.off("verification.summary", onVerificationSummary);
client.off("room.invite", onInvite);
client.off("room.join", onJoin);
client.off("room.event", onRoomEvent);
};
}
@@ -0,0 +1,441 @@
import { vi } from "vitest";
import { z } from "zod";
import type { MatrixRoomInfo } from "./room-info.js";
export type DirectRoomTrackerOptions = {
isExplicitlyConfiguredRoom?: (roomId: string) => boolean | Promise<boolean>;
canPromoteRecentInvite?: (roomId: string) => boolean | Promise<boolean>;
canPromoteUnmappedStrictRoom?: (roomId: string) => boolean | Promise<boolean>;
shouldKeepLocallyPromotedDirectRoom?:
| ((roomId: string) => boolean | undefined | Promise<boolean | undefined>)
| undefined;
};
type MonitorRetirement = {
closeTaskAdmission: () => void;
detachListeners: () => void;
waitForTasks: () => Promise<void>;
cleanup: () => Promise<void> | void;
};
const hoisted = vi.hoisted(() => {
const createEmitter = () => {
const listeners = new Map<string, Set<(...args: unknown[]) => void>>();
return {
on(event: string, listener: (...args: unknown[]) => void) {
let bucket = listeners.get(event);
if (!bucket) {
bucket = new Set();
listeners.set(event, bucket);
}
bucket.add(listener);
return this;
},
off(event: string, listener: (...args: unknown[]) => void) {
listeners.get(event)?.delete(listener);
return this;
},
emit(event: string, ...args: unknown[]) {
for (const listener of listeners.get(event) ?? []) {
listener(...args);
}
return true;
},
listenerCount(event: string) {
return listeners.get(event)?.size ?? 0;
},
removeAllListeners() {
listeners.clear();
return this;
},
};
};
const callOrder: string[] = [];
const state = {
leaseAbortController: new AbortController(),
monitorRetirement: null as MonitorRetirement | null,
monitorRetirementPromise: null as Promise<void> | null,
startClientError: null as Error | null,
};
const accountConfig = {
dm: {},
};
const inboundReplayClaim = {
keys: ["test"] as const,
commit: vi.fn(async () => true),
release: vi.fn(),
};
const inboundDeduper = {
claim: vi.fn(async () => ({ kind: "claimed" as const, handle: inboundReplayClaim })),
};
const createMatrixInboundEventDeduper = vi.fn(() => inboundDeduper);
const client = Object.assign(createEmitter(), {
id: "matrix-client",
hasPersistedSyncState: vi.fn(() => false),
drainPendingDecryptions: vi.fn(async () => undefined),
});
const createMatrixRoomMessageHandler = vi.fn(() => vi.fn());
const createDirectRoomTracker = vi.fn(
(_clientForTest: unknown, _opts?: DirectRoomTrackerOptions) => ({
isDirectMessage: vi.fn(async () => false),
}),
);
const getRoomInfo = vi.fn<
(roomId: string, opts?: { includeAliases?: boolean }) => Promise<MatrixRoomInfo>
>(async () => ({
altAliases: [],
nameResolved: true,
aliasesResolved: true,
}));
const getMemberDisplayName = vi.fn(async () => "Bot");
const resolveTextChunkLimit = vi.fn<
(cfg: unknown, channel: unknown, accountId?: unknown) => number
>(() => 4000);
const logger = {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
};
const stopThreadBindingManager = vi.fn();
const createThreadBindingManager = vi.fn(async () => {
callOrder.push("create-manager");
return {
accountId: "default",
stop: stopThreadBindingManager,
};
});
const disposeAutoJoin = vi.fn(() => {
callOrder.push("dispose-auto-join");
});
const disposeMonitorEvents = vi.fn(() => {
callOrder.push("dispose-monitor-events");
});
const acquireSharedMatrixClientImpl = async (params: { startClient?: boolean }) => {
if (params.startClient !== false) {
throw new Error("Matrix monitor must acquire its lease before startup");
}
callOrder.push("prepare-client");
return lease;
};
const registerMonitorRetirement = vi.fn((retirement: MonitorRetirement) => {
state.monitorRetirement = retirement;
state.monitorRetirementPromise = null;
});
const runRegisteredMonitorRetirement = (): Promise<void> => {
if (state.monitorRetirementPromise) {
return state.monitorRetirementPromise;
}
state.monitorRetirementPromise = (async () => {
const retirement = state.monitorRetirement;
retirement?.closeTaskAdmission();
retirement?.detachListeners();
await retirement?.waitForTasks();
await retirement?.cleanup();
})();
return state.monitorRetirementPromise;
};
const finalReleaseImpl = async () => {
await runRegisteredMonitorRetirement();
};
const resolveSharedMatrixClientImpl = async () => {
if (!callOrder.includes("create-manager")) {
throw new Error("Matrix client started before thread bindings were registered");
}
if (!state.monitorRetirement) {
throw new Error("Matrix client started before monitor retirement was registered");
}
if (state.startClientError) {
throw state.startClientError;
}
callOrder.push("start-client");
return client;
};
const acquireSharedMatrixClient = vi.fn(acquireSharedMatrixClientImpl);
const releaseSharedClientInstance = vi.fn(finalReleaseImpl);
const lease = {
get abortSignal() {
return state.leaseAbortController.signal;
},
client,
role: "monitor" as const,
registerMonitorRetirement,
start: vi.fn(resolveSharedMatrixClientImpl),
release: releaseSharedClientInstance,
};
const registerMatrixAutoJoin = vi.fn(() => disposeAutoJoin);
const registerMatrixMonitorEvents = vi.fn(
(_params: {
getHealthySyncSinceMs?: () => number | undefined;
onRoomMessage: (roomId: string, event: unknown) => Promise<void>;
runDetachedTask?: (label: string, task: () => Promise<void>) => Promise<void>;
}) => {
callOrder.push("register-events");
return disposeMonitorEvents;
},
);
const registerChannelRuntimeContext = vi.fn();
const setMatrixRuntime = vi.fn();
const backfillMatrixAuthDeviceIdAfterStartup = vi.fn(async () => undefined);
const runMatrixStartupMaintenance = vi.fn<
(params: { abortSignal?: AbortSignal }) => Promise<void>
>(async () => undefined);
const setStatus = vi.fn();
return {
accountConfig,
acquireSharedMatrixClient,
acquireSharedMatrixClientImpl,
backfillMatrixAuthDeviceIdAfterStartup,
callOrder,
client,
createDirectRoomTracker,
createMatrixInboundEventDeduper,
createMatrixRoomMessageHandler,
createThreadBindingManager,
disposeAutoJoin,
disposeMonitorEvents,
finalReleaseImpl,
getMemberDisplayName,
getRoomInfo,
inboundDeduper,
inboundReplayClaim,
logger,
registeredHealthySyncGetter: undefined as undefined | (() => number | undefined),
registeredOnRoomMessage: null as null | ((roomId: string, event: unknown) => Promise<void>),
registerChannelRuntimeContext,
registerMatrixAutoJoin,
registerMatrixMonitorEvents,
registerMonitorRetirement,
releaseSharedClientInstance,
resolveSharedMatrixClient: lease.start,
resolveSharedMatrixClientImpl,
resolveTextChunkLimit,
runMatrixStartupMaintenance,
runRegisteredMonitorRetirement,
setMatrixRuntime,
setStatus,
state,
stopThreadBindingManager,
};
});
vi.mock("openclaw/plugin-sdk/channel-runtime-context", () => ({
registerChannelRuntimeContext: hoisted.registerChannelRuntimeContext,
}));
vi.mock("../../runtime-api.js", () => {
const normalizeAccountId = (value: string | null | undefined) => value?.trim() || "default";
return {
DEFAULT_ACCOUNT_ID: "default",
GROUP_POLICY_BLOCKED_LABEL: {
room: "room",
},
MarkdownConfigSchema: z.any().optional(),
PAIRING_APPROVED_MESSAGE: "paired",
ToolPolicySchema: z.any().optional(),
addAllowlistUserEntriesFromConfigEntry: vi.fn(),
buildChannelConfigSchema: (schema: unknown) => schema,
buildChannelKeyCandidates: (...keys: Array<string | undefined | null>) => {
const seen = new Set<string>();
return keys
.map((key) => (typeof key === "string" ? key.trim() : ""))
.filter((key) => {
if (!key || seen.has(key)) {
return false;
}
seen.add(key);
return true;
});
},
buildProbeChannelStatusSummary: (
snapshot: Record<string, unknown>,
extra?: Record<string, unknown>,
) => ({
...snapshot,
...extra,
}),
buildSecretInputSchema: () => z.string(),
chunkTextForOutbound: vi.fn((text: string) => [text]),
collectStatusIssuesFromLastError: () => [],
createActionGate: () => () => true,
createReplyPrefixOptions: () => ({}),
createTypingCallbacks: () => ({}),
formatDocsLink: (input: string) => input,
formatZonedTimestamp: () => "2026-03-27T00:00:00.000Z",
getAgentScopedMediaLocalRoots: () => [],
getSessionBindingService: () => ({}),
hasConfiguredSecretInput: (value: unknown) => Boolean(value),
mergeAllowlist: ({ existing, additions }: { existing: string[]; additions: string[] }) => [
...existing,
...additions,
],
normalizeAccountId,
normalizeOptionalAccountId: normalizeAccountId,
resolveThreadBindingIdleTimeoutMsForChannel: () => 24 * 60 * 60 * 1000,
resolveThreadBindingMaxAgeMsForChannel: () => 0,
resolveAllowlistProviderRuntimeGroupPolicy: () => ({
groupPolicy: "allowlist",
providerMissingFallbackApplied: false,
}),
resolveDefaultGroupPolicy: () => "allowlist",
resolveOutboundSendDep: () => null,
resolveThreadBindingFarewellText: () => null,
resolveAckReaction: () => null,
readJsonFileWithFallback: vi.fn(),
readNumberParam: vi.fn(),
readReactionParams: vi.fn(),
readStringArrayParam: vi.fn(),
readStringParam: vi.fn(),
summarizeMapping: vi.fn(),
warnMissingProviderGroupPolicyFallbackOnce: vi.fn(),
};
});
vi.mock("../../resolve-targets.js", () => ({
resolveMatrixTargets: vi.fn(async () => []),
}));
vi.mock("../../runtime.js", () => ({
getMatrixRuntime: () => ({
config: {
current: () => ({
channels: {
matrix: hoisted.accountConfig,
},
}),
replaceConfigFile: vi.fn(),
mutateConfigFile: vi.fn(),
},
logging: {
getChildLogger: () => hoisted.logger,
shouldLogVerbose: () => false,
},
channel: {
mentions: {
buildMentionRegexes: () => [],
},
text: {
resolveTextChunkLimit: (cfg: unknown, channel: unknown, accountId?: unknown) =>
hoisted.resolveTextChunkLimit(cfg, channel, accountId),
},
},
system: {
formatNativeDependencyHint: () => "",
},
media: {
loadWebMedia: vi.fn(),
},
}),
setMatrixRuntime: hoisted.setMatrixRuntime,
}));
vi.mock("../accounts.js", async () => {
const actual = await vi.importActual<typeof import("../accounts.js")>("../accounts.js");
return {
...actual,
resolveConfiguredMatrixBotUserIds: vi.fn(() => new Set<string>()),
resolveMatrixAccount: () => ({
accountId: "default",
config: hoisted.accountConfig,
}),
};
});
vi.mock("../client.js", () => ({
acquireSharedMatrixClient: hoisted.acquireSharedMatrixClient,
backfillMatrixAuthDeviceIdAfterStartup: hoisted.backfillMatrixAuthDeviceIdAfterStartup,
isBunRuntime: () => false,
resolveMatrixAuth: vi.fn(async () => ({
accountId: "default",
homeserver: "https://matrix.example.org",
userId: "@bot:example.org",
accessToken: "token",
initialSyncLimit: 20,
encryption: false,
})),
resolveMatrixAuthContext: vi.fn(() => ({
accountId: "default",
})),
}));
vi.mock("../config-update.js", () => ({
updateMatrixAccountConfig: vi.fn((cfg: unknown) => cfg),
}));
vi.mock("../device-health.js", () => ({
summarizeMatrixDeviceHealth: vi.fn(() => ({
staleOpenClawDevices: [],
})),
}));
vi.mock("../profile.js", () => ({
syncMatrixOwnProfile: vi.fn(async () => ({
displayNameUpdated: false,
avatarUpdated: false,
convertedAvatarFromHttp: false,
resolvedAvatarUrl: undefined,
})),
}));
vi.mock("../thread-bindings.js", () => ({
createMatrixThreadBindingManager: hoisted.createThreadBindingManager,
}));
vi.mock("./allowlist.js", () => ({
normalizeMatrixUserId: (value: string) => value,
}));
vi.mock("./auto-join.js", () => ({
registerMatrixAutoJoin: hoisted.registerMatrixAutoJoin,
}));
vi.mock("./direct.js", () => ({
createDirectRoomTracker: hoisted.createDirectRoomTracker,
}));
vi.mock("./events.js", () => ({
registerMatrixMonitorEvents: hoisted.registerMatrixMonitorEvents.mockImplementation(
(params: {
getHealthySyncSinceMs?: () => number | undefined;
onRoomMessage: (roomId: string, event: unknown) => Promise<void>;
runDetachedTask?: (label: string, task: () => Promise<void>) => Promise<void>;
}) => {
hoisted.callOrder.push("register-events");
hoisted.registeredHealthySyncGetter = params.getHealthySyncSinceMs;
hoisted.registeredOnRoomMessage = (roomId: string, event: unknown) =>
params.runDetachedTask
? params.runDetachedTask("test room message", async () => {
await params.onRoomMessage(roomId, event);
})
: params.onRoomMessage(roomId, event);
return hoisted.disposeMonitorEvents;
},
),
}));
vi.mock("./handler.js", () => ({
createMatrixRoomMessageHandler: hoisted.createMatrixRoomMessageHandler,
}));
vi.mock("./inbound-dedupe.js", () => ({
createMatrixInboundEventDeduper: hoisted.createMatrixInboundEventDeduper,
}));
vi.mock("./room-info.js", () => ({
createMatrixRoomInfoResolver: vi.fn(() => ({
getRoomInfo: hoisted.getRoomInfo,
getMemberDisplayName: hoisted.getMemberDisplayName,
})),
}));
vi.mock("./startup-verification.js", () => ({
ensureMatrixStartupVerification: vi.fn(),
}));
vi.mock("./startup.js", () => ({
runMatrixStartupMaintenance: hoisted.runMatrixStartupMaintenance,
}));
export function getMatrixMonitorIndexTestHarness() {
return hoisted;
}
+225 -444
View File
@@ -1,367 +1,12 @@
// Matrix tests cover index plugin behavior.
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { z } from "zod";
import type { MatrixConfig, MatrixStreamingMode } from "../../types.js";
import type { MatrixRoomInfo } from "./room-info.js";
import {
getMatrixMonitorIndexTestHarness,
type DirectRoomTrackerOptions,
} from "./index.test-helpers.js";
type DirectRoomTrackerOptions = {
isExplicitlyConfiguredRoom?: (roomId: string) => boolean | Promise<boolean>;
canPromoteRecentInvite?: (roomId: string) => boolean | Promise<boolean>;
canPromoteUnmappedStrictRoom?: (roomId: string) => boolean | Promise<boolean>;
shouldKeepLocallyPromotedDirectRoom?:
| ((roomId: string) => boolean | undefined | Promise<boolean | undefined>)
| undefined;
};
const hoisted = vi.hoisted(() => {
const createEmitter = () => {
const listeners = new Map<string, Set<(...args: unknown[]) => void>>();
return {
on(event: string, listener: (...args: unknown[]) => void) {
let bucket = listeners.get(event);
if (!bucket) {
bucket = new Set();
listeners.set(event, bucket);
}
bucket.add(listener);
return this;
},
off(event: string, listener: (...args: unknown[]) => void) {
listeners.get(event)?.delete(listener);
return this;
},
emit(event: string, ...args: unknown[]) {
for (const listener of listeners.get(event) ?? []) {
listener(...args);
}
return true;
},
removeAllListeners() {
listeners.clear();
return this;
},
};
};
const callOrder: string[] = [];
const state = {
startClientError: null as Error | null,
};
const accountConfig = {
dm: {},
};
const inboundReplayClaim = {
keys: ["test"] as const,
commit: vi.fn(async () => true),
release: vi.fn(),
};
const inboundDeduper = {
claim: vi.fn(async () => ({ kind: "claimed" as const, handle: inboundReplayClaim })),
};
const createMatrixInboundEventDeduper = vi.fn(() => inboundDeduper);
const client = Object.assign(createEmitter(), {
id: "matrix-client",
hasPersistedSyncState: vi.fn(() => false),
stopSyncWithoutPersist: vi.fn(),
drainPendingDecryptions: vi.fn(async () => undefined),
});
const createMatrixRoomMessageHandler = vi.fn(() => vi.fn());
const createDirectRoomTracker = vi.fn(
(_clientForTest: unknown, _opts?: DirectRoomTrackerOptions) => ({
isDirectMessage: vi.fn(async () => false),
}),
);
const getRoomInfo = vi.fn<
(roomId: string, opts?: { includeAliases?: boolean }) => Promise<MatrixRoomInfo>
>(async () => ({
altAliases: [],
nameResolved: true,
aliasesResolved: true,
}));
const getMemberDisplayName = vi.fn(async () => "Bot");
const resolveTextChunkLimit = vi.fn<
(cfg: unknown, channel: unknown, accountId?: unknown) => number
>(() => 4000);
const logger = {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
};
const stopThreadBindingManager = vi.fn();
const releaseSharedClientInstance = vi.fn(async () => true);
const resolveSharedMatrixClient = vi.fn(async (params: { startClient?: boolean }) => {
if (params.startClient === false) {
callOrder.push("prepare-client");
return client;
}
if (!callOrder.includes("create-manager")) {
throw new Error("Matrix client started before thread bindings were registered");
}
if (state.startClientError) {
throw state.startClientError;
}
callOrder.push("start-client");
return client;
});
const setActiveMatrixClient = vi.fn();
const setMatrixRuntime = vi.fn();
const backfillMatrixAuthDeviceIdAfterStartup = vi.fn(async () => undefined);
const runMatrixStartupMaintenance = vi.fn<
(params: { abortSignal?: AbortSignal }) => Promise<void>
>(async () => undefined);
const setStatus = vi.fn();
return {
backfillMatrixAuthDeviceIdAfterStartup,
callOrder,
accountConfig,
client,
createDirectRoomTracker,
createMatrixInboundEventDeduper,
createMatrixRoomMessageHandler,
getMemberDisplayName,
getRoomInfo,
inboundDeduper,
inboundReplayClaim,
logger,
registeredOnRoomMessage: null as null | ((roomId: string, event: unknown) => Promise<void>),
releaseSharedClientInstance,
resolveSharedMatrixClient,
resolveTextChunkLimit,
runMatrixStartupMaintenance,
registeredHealthySyncGetter: undefined as undefined | (() => number | undefined),
setActiveMatrixClient,
setMatrixRuntime,
setStatus,
state,
stopThreadBindingManager,
};
});
vi.mock("../../runtime-api.js", () => {
const normalizeAccountId = (value: string | null | undefined) => value?.trim() || "default";
return {
DEFAULT_ACCOUNT_ID: "default",
GROUP_POLICY_BLOCKED_LABEL: {
room: "room",
},
MarkdownConfigSchema: z.any().optional(),
PAIRING_APPROVED_MESSAGE: "paired",
ToolPolicySchema: z.any().optional(),
addAllowlistUserEntriesFromConfigEntry: vi.fn(),
buildChannelConfigSchema: (schema: unknown) => schema,
buildChannelKeyCandidates: (...keys: Array<string | undefined | null>) => {
const seen = new Set<string>();
return keys
.map((key) => (typeof key === "string" ? key.trim() : ""))
.filter((key) => {
if (!key || seen.has(key)) {
return false;
}
seen.add(key);
return true;
});
},
buildProbeChannelStatusSummary: (
snapshot: Record<string, unknown>,
extra?: Record<string, unknown>,
) => ({
...snapshot,
...extra,
}),
buildSecretInputSchema: () => z.string(),
chunkTextForOutbound: vi.fn((text: string) => [text]),
collectStatusIssuesFromLastError: () => [],
createActionGate: () => () => true,
createReplyPrefixOptions: () => ({}),
createTypingCallbacks: () => ({}),
formatDocsLink: (input: string) => input,
formatZonedTimestamp: () => "2026-03-27T00:00:00.000Z",
getAgentScopedMediaLocalRoots: () => [],
getSessionBindingService: () => ({}),
hasConfiguredSecretInput: (value: unknown) => Boolean(value),
mergeAllowlist: ({ existing, additions }: { existing: string[]; additions: string[] }) => [
...existing,
...additions,
],
normalizeAccountId,
normalizeOptionalAccountId: normalizeAccountId,
resolveThreadBindingIdleTimeoutMsForChannel: () => 24 * 60 * 60 * 1000,
resolveThreadBindingMaxAgeMsForChannel: () => 0,
resolveAllowlistProviderRuntimeGroupPolicy: () => ({
groupPolicy: "allowlist",
providerMissingFallbackApplied: false,
}),
resolveDefaultGroupPolicy: () => "allowlist",
resolveOutboundSendDep: () => null,
resolveThreadBindingFarewellText: () => null,
resolveAckReaction: () => null,
readJsonFileWithFallback: vi.fn(),
readNumberParam: vi.fn(),
readReactionParams: vi.fn(),
readStringArrayParam: vi.fn(),
readStringParam: vi.fn(),
summarizeMapping: vi.fn(),
warnMissingProviderGroupPolicyFallbackOnce: vi.fn(),
};
});
vi.mock("../../resolve-targets.js", () => ({
resolveMatrixTargets: vi.fn(async () => []),
}));
vi.mock("../../runtime.js", () => ({
getMatrixRuntime: () => ({
config: {
current: () => ({
channels: {
matrix: hoisted.accountConfig,
},
}),
replaceConfigFile: vi.fn(),
mutateConfigFile: vi.fn(),
},
logging: {
getChildLogger: () => hoisted.logger,
shouldLogVerbose: () => false,
},
channel: {
mentions: {
buildMentionRegexes: () => [],
},
text: {
resolveTextChunkLimit: (cfg: unknown, channel: unknown, accountId?: unknown) =>
hoisted.resolveTextChunkLimit(cfg, channel, accountId),
},
},
system: {
formatNativeDependencyHint: () => "",
},
media: {
loadWebMedia: vi.fn(),
},
}),
setMatrixRuntime: hoisted.setMatrixRuntime,
}));
vi.mock("../accounts.js", async () => {
const actual = await vi.importActual<typeof import("../accounts.js")>("../accounts.js");
return {
...actual,
resolveConfiguredMatrixBotUserIds: vi.fn(() => new Set<string>()),
resolveMatrixAccount: () => ({
accountId: "default",
config: hoisted.accountConfig,
}),
};
});
vi.mock("../active-client.js", () => ({
setActiveMatrixClient: hoisted.setActiveMatrixClient,
}));
vi.mock("../client.js", () => ({
backfillMatrixAuthDeviceIdAfterStartup: hoisted.backfillMatrixAuthDeviceIdAfterStartup,
isBunRuntime: () => false,
resolveMatrixAuth: vi.fn(async () => ({
accountId: "default",
homeserver: "https://matrix.example.org",
userId: "@bot:example.org",
accessToken: "token",
initialSyncLimit: 20,
encryption: false,
})),
resolveMatrixAuthContext: vi.fn(() => ({
accountId: "default",
})),
resolveSharedMatrixClient: hoisted.resolveSharedMatrixClient,
}));
vi.mock("../client/shared.js", () => ({
releaseSharedClientInstance: hoisted.releaseSharedClientInstance,
}));
vi.mock("../config-update.js", () => ({
updateMatrixAccountConfig: vi.fn((cfg: unknown) => cfg),
}));
vi.mock("../device-health.js", () => ({
summarizeMatrixDeviceHealth: vi.fn(() => ({
staleOpenClawDevices: [],
})),
}));
vi.mock("../profile.js", () => ({
syncMatrixOwnProfile: vi.fn(async () => ({
displayNameUpdated: false,
avatarUpdated: false,
convertedAvatarFromHttp: false,
resolvedAvatarUrl: undefined,
})),
}));
vi.mock("../thread-bindings.js", () => ({
createMatrixThreadBindingManager: vi.fn(async () => {
hoisted.callOrder.push("create-manager");
return {
accountId: "default",
stop: hoisted.stopThreadBindingManager,
};
}),
}));
vi.mock("./allowlist.js", () => ({
normalizeMatrixUserId: (value: string) => value,
}));
vi.mock("./auto-join.js", () => ({
registerMatrixAutoJoin: vi.fn(),
}));
vi.mock("./direct.js", () => ({
createDirectRoomTracker: hoisted.createDirectRoomTracker,
}));
vi.mock("./events.js", () => ({
registerMatrixMonitorEvents: vi.fn(
(params: {
getHealthySyncSinceMs?: () => number | undefined;
onRoomMessage: (roomId: string, event: unknown) => Promise<void>;
runDetachedTask?: (label: string, task: () => Promise<void>) => Promise<void>;
}) => {
hoisted.callOrder.push("register-events");
hoisted.registeredHealthySyncGetter = params.getHealthySyncSinceMs;
hoisted.registeredOnRoomMessage = (roomId: string, event: unknown) =>
params.runDetachedTask
? params.runDetachedTask("test room message", async () => {
await params.onRoomMessage(roomId, event);
})
: params.onRoomMessage(roomId, event);
},
),
}));
vi.mock("./handler.js", () => ({
createMatrixRoomMessageHandler: hoisted.createMatrixRoomMessageHandler,
}));
vi.mock("./inbound-dedupe.js", () => ({
createMatrixInboundEventDeduper: hoisted.createMatrixInboundEventDeduper,
}));
vi.mock("./room-info.js", () => ({
createMatrixRoomInfoResolver: vi.fn(() => ({
getRoomInfo: hoisted.getRoomInfo,
getMemberDisplayName: hoisted.getMemberDisplayName,
})),
}));
vi.mock("./startup-verification.js", () => ({
ensureMatrixStartupVerification: vi.fn(),
}));
vi.mock("./startup.js", () => ({
runMatrixStartupMaintenance: hoisted.runMatrixStartupMaintenance,
}));
const hoisted = getMatrixMonitorIndexTestHarness();
let monitorMatrixProvider: typeof import("./index.js").monitorMatrixProvider;
@@ -395,6 +40,20 @@ describe("monitorMatrixProvider", () => {
await monitorPromise;
}
function registeredRoomMessageHandler() {
const handler = hoisted.registeredOnRoomMessage;
if (!handler) {
throw new Error("expected room message handler to be registered");
}
return handler;
}
function expectPersistRelease() {
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(
expect.objectContaining({ mode: "persist" }),
);
}
function mockCallArg(mock: { mock: { calls: unknown[][] } }, index = 0, argIndex = 0): unknown {
const call = mock.mock.calls.at(index);
if (!call) {
@@ -436,31 +95,32 @@ describe("monitorMatrixProvider", () => {
beforeEach(() => {
hoisted.callOrder.length = 0;
hoisted.state.leaseAbortController = new AbortController();
hoisted.state.monitorRetirement = null;
hoisted.state.monitorRetirementPromise = null;
hoisted.state.startClientError = null;
hoisted.accountConfig.dm = {};
delete (hoisted.accountConfig as { streaming?: unknown }).streaming;
delete (hoisted.accountConfig as { rooms?: Record<string, unknown> }).rooms;
hoisted.resolveTextChunkLimit.mockReset().mockReturnValue(4000);
hoisted.releaseSharedClientInstance.mockReset().mockResolvedValue(true);
hoisted.acquireSharedMatrixClient
.mockReset()
.mockImplementation(hoisted.acquireSharedMatrixClientImpl);
hoisted.releaseSharedClientInstance.mockReset().mockImplementation(hoisted.finalReleaseImpl);
hoisted.registerMonitorRetirement.mockClear();
hoisted.resolveSharedMatrixClient
.mockReset()
.mockImplementation(async (params: { startClient?: boolean }) => {
if (params.startClient === false) {
hoisted.callOrder.push("prepare-client");
return hoisted.client;
}
if (!hoisted.callOrder.includes("create-manager")) {
throw new Error("Matrix client started before thread bindings were registered");
}
if (hoisted.state.startClientError) {
throw hoisted.state.startClientError;
}
hoisted.callOrder.push("start-client");
return hoisted.client;
});
.mockImplementation(hoisted.resolveSharedMatrixClientImpl);
hoisted.createDirectRoomTracker.mockReset().mockReturnValue({
isDirectMessage: vi.fn(async () => false),
});
hoisted.createThreadBindingManager.mockReset().mockImplementation(async () => {
hoisted.callOrder.push("create-manager");
return {
accountId: "default",
stop: hoisted.stopThreadBindingManager,
};
});
hoisted.getRoomInfo.mockReset().mockResolvedValue({
altAliases: [],
nameResolved: true,
@@ -469,11 +129,9 @@ describe("monitorMatrixProvider", () => {
hoisted.getMemberDisplayName.mockReset().mockResolvedValue("Bot");
hoisted.registeredOnRoomMessage = null;
hoisted.registeredHealthySyncGetter = undefined;
hoisted.setActiveMatrixClient.mockReset();
hoisted.stopThreadBindingManager.mockReset();
hoisted.client.removeAllListeners();
hoisted.client.hasPersistedSyncState.mockReset().mockReturnValue(false);
hoisted.client.stopSyncWithoutPersist.mockReset();
hoisted.client.drainPendingDecryptions.mockReset().mockResolvedValue(undefined);
hoisted.inboundDeduper.claim
.mockReset()
@@ -482,8 +140,13 @@ describe("monitorMatrixProvider", () => {
hoisted.inboundReplayClaim.release.mockReset();
hoisted.createMatrixInboundEventDeduper.mockReset().mockReturnValue(hoisted.inboundDeduper);
hoisted.backfillMatrixAuthDeviceIdAfterStartup.mockReset().mockResolvedValue(undefined);
hoisted.registerChannelRuntimeContext.mockReset();
hoisted.runMatrixStartupMaintenance.mockReset().mockResolvedValue(undefined);
hoisted.createMatrixRoomMessageHandler.mockReset().mockReturnValue(vi.fn());
hoisted.disposeAutoJoin.mockClear();
hoisted.disposeMonitorEvents.mockClear();
hoisted.registerMatrixAutoJoin.mockClear();
hoisted.registerMatrixMonitorEvents.mockClear();
hoisted.setStatus.mockReset();
Object.values(hoisted.logger).forEach((mock) => mock.mockReset());
});
@@ -531,7 +194,7 @@ describe("monitorMatrixProvider", () => {
expect(hoisted.callOrder).toStrictEqual([]);
expect(hoisted.resolveTextChunkLimit).not.toHaveBeenCalled();
expect(hoisted.createMatrixRoomMessageHandler).not.toHaveBeenCalled();
expect(hoisted.setActiveMatrixClient).not.toHaveBeenCalled();
expect(hoisted.acquireSharedMatrixClient).not.toHaveBeenCalled();
});
it("publishes disconnected startup status and connected sync status without failing the monitor", async () => {
@@ -631,10 +294,7 @@ describe("monitorMatrixProvider", () => {
const monitorPromise = monitorMatrixProvider({ abortSignal: abortController.signal });
await waitForCallOrderEntry("start-client");
const onRoomMessage = hoisted.registeredOnRoomMessage;
if (!onRoomMessage) {
throw new Error("expected room message handler to be registered");
}
const onRoomMessage = registeredRoomMessageHandler();
await onRoomMessage("!room:example.org", { event_id: "$event" });
await Promise.resolve();
@@ -664,7 +324,7 @@ describe("monitorMatrixProvider", () => {
hoisted.client.emit("sync.unexpected_error", new Error("sync exploded"));
await expect(monitorPromise).rejects.toThrow("sync exploded");
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(hoisted.client, "persist");
expectPersistRelease();
expectStatusCallFields({
accountId: "default",
connected: false,
@@ -674,15 +334,7 @@ describe("monitorMatrixProvider", () => {
});
it("marks early startup failures as error before the monitor loop starts", async () => {
hoisted.resolveSharedMatrixClient.mockImplementation(
async (params: { startClient?: boolean }) => {
if (params.startClient === false) {
throw new Error("prepare failed");
}
hoisted.callOrder.push("start-client");
return hoisted.client;
},
);
hoisted.acquireSharedMatrixClient.mockRejectedValue(new Error("prepare failed"));
await expect(
monitorMatrixProvider({
@@ -710,7 +362,7 @@ describe("monitorMatrixProvider", () => {
}),
).rejects.toThrow("deduper failed");
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(hoisted.client, "persist");
expectPersistRelease();
expectLastStatusFields({
accountId: "default",
connected: false,
@@ -721,26 +373,20 @@ describe("monitorMatrixProvider", () => {
it("aborts stalled startup promptly and releases the shared client without persist", async () => {
const abortController = new AbortController();
hoisted.resolveSharedMatrixClient.mockImplementation(
async (params: { startClient?: boolean; abortSignal?: AbortSignal }) => {
if (params.startClient === false) {
hoisted.callOrder.push("prepare-client");
return hoisted.client;
}
hoisted.callOrder.push("start-client");
return await new Promise<typeof hoisted.client>((_resolve, reject) => {
params.abortSignal?.addEventListener(
"abort",
() => {
const error = new Error("Matrix startup aborted");
error.name = "AbortError";
reject(error);
},
{ once: true },
);
});
},
);
hoisted.resolveSharedMatrixClient.mockImplementation(async (abortSignal?: AbortSignal) => {
hoisted.callOrder.push("start-client");
return await new Promise<typeof hoisted.client>((_resolve, reject) => {
abortSignal?.addEventListener(
"abort",
() => {
const error = new Error("Matrix startup aborted");
error.name = "AbortError";
reject(error);
},
{ once: true },
);
});
});
const monitorPromise = monitorMatrixProvider({ abortSignal: abortController.signal });
@@ -749,12 +395,16 @@ describe("monitorMatrixProvider", () => {
abortController.abort();
await expect(monitorPromise).resolves.toBeUndefined();
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(hoisted.client, "stop");
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(
expect.objectContaining({ mode: "stop" }),
);
expect(hoisted.client.drainPendingDecryptions).not.toHaveBeenCalled();
});
it("aborts during startup maintenance and releases the shared client without persist", async () => {
const abortController = new AbortController();
const handler = vi.fn(async () => {});
hoisted.createMatrixRoomMessageHandler.mockReturnValue(handler);
hoisted.runMatrixStartupMaintenance.mockImplementation(
async (params: { abortSignal?: AbortSignal }) =>
await new Promise<void>((_resolve, reject) => {
@@ -780,8 +430,53 @@ describe("monitorMatrixProvider", () => {
abortController.abort();
await expect(monitorPromise).resolves.toBeUndefined();
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(hoisted.client, "stop");
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(
expect.objectContaining({ mode: "stop" }),
);
expect(hoisted.client.drainPendingDecryptions).not.toHaveBeenCalled();
await hoisted.registeredOnRoomMessage?.("!room:example.org", { event_id: "$late" });
expect(handler).not.toHaveBeenCalled();
});
it("disposes resources installed after forced retirement wins setup", async () => {
const stopLateManager = vi.fn();
let resolveManager: (manager: {
accountId: string;
stop: typeof stopLateManager;
}) => void = () => {};
const managerReady = new Promise<{
accountId: string;
stop: typeof stopLateManager;
}>((resolve) => {
resolveManager = resolve;
});
hoisted.createThreadBindingManager.mockImplementation(async () => {
hoisted.callOrder.push("create-manager");
return await managerReady;
});
const monitorPromise = monitorMatrixProvider();
await waitForCallOrderEntry("create-manager");
await hoisted.runRegisteredMonitorRetirement();
expect(hoisted.disposeAutoJoin).toHaveBeenCalledTimes(1);
expect(hoisted.client.listenerCount("sync.state")).toBe(0);
expect(hoisted.registerMatrixMonitorEvents).not.toHaveBeenCalled();
resolveManager({
accountId: "default",
stop: stopLateManager,
});
await expect(monitorPromise).resolves.toBeUndefined();
expect(stopLateManager).toHaveBeenCalledTimes(1);
expect(hoisted.resolveSharedMatrixClient).not.toHaveBeenCalled();
expect(hoisted.registerMatrixMonitorEvents).not.toHaveBeenCalled();
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(
expect.objectContaining({ mode: "stop" }),
);
expect(hoisted.disposeAutoJoin).toHaveBeenCalledTimes(1);
expect(hoisted.client.listenerCount("sync.state")).toBe(0);
});
it("registers Matrix thread bindings before starting the client", async () => {
@@ -792,7 +487,12 @@ describe("monitorMatrixProvider", () => {
"create-manager",
"register-events",
"start-client",
"dispose-auto-join",
"dispose-monitor-events",
]);
expect(hoisted.acquireSharedMatrixClient).toHaveBeenCalledWith(
expect.objectContaining({ startClient: false }),
);
expect(hoisted.stopThreadBindingManager).toHaveBeenCalledTimes(1);
});
@@ -826,12 +526,49 @@ describe("monitorMatrixProvider", () => {
const backfillParams = mockCallArg(hoisted.backfillMatrixAuthDeviceIdAfterStartup) as {
abortSignal?: AbortSignal;
};
expect(backfillParams.abortSignal).toBe(abortController.signal);
expect(backfillParams.abortSignal).not.toBe(abortController.signal);
expect(backfillParams.abortSignal?.aborted).toBe(false);
abortController.abort();
expect(backfillParams.abortSignal?.aborted).toBe(true);
await expect(monitorPromise).resolves.toBeUndefined();
});
it("terminates a fully started monitor when forced retirement aborts its lease", async () => {
const monitorPromise = monitorMatrixProvider();
await flushUntil(
() =>
hoisted.runMatrixStartupMaintenance.mock.calls.length === 1 &&
hoisted.registerChannelRuntimeContext.mock.calls.length === 1,
"expected monitor startup consumers to be registered",
);
const startSignal = mockCallArg(hoisted.resolveSharedMatrixClient, 0, 0) as AbortSignal;
const backfillParams = mockCallArg(hoisted.backfillMatrixAuthDeviceIdAfterStartup) as {
abortSignal?: AbortSignal;
};
const runtimeContextParams = mockCallArg(hoisted.registerChannelRuntimeContext) as {
abortSignal?: AbortSignal;
};
const maintenanceParams = mockCallArg(hoisted.runMatrixStartupMaintenance) as {
abortSignal?: AbortSignal;
};
expect(startSignal).toBe(hoisted.state.leaseAbortController.signal);
expect(backfillParams.abortSignal).toBe(startSignal);
expect(runtimeContextParams.abortSignal).toBe(startSignal);
expect(maintenanceParams.abortSignal).toBe(startSignal);
expect(startSignal.aborted).toBe(false);
hoisted.state.leaseAbortController.abort();
await hoisted.runRegisteredMonitorRetirement();
await expect(monitorPromise).resolves.toBeUndefined();
expect(startSignal.aborted).toBe(true);
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledTimes(1);
expect(hoisted.stopThreadBindingManager).toHaveBeenCalledTimes(1);
expect(hoisted.client.listenerCount("sync.state")).toBe(0);
});
it("cleans up thread bindings and shared clients when startup fails", async () => {
hoisted.state.startClientError = new Error("start failed");
@@ -839,9 +576,7 @@ describe("monitorMatrixProvider", () => {
expect(hoisted.stopThreadBindingManager).toHaveBeenCalledTimes(1);
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledTimes(1);
expect(hoisted.releaseSharedClientInstance).toHaveBeenCalledWith(hoisted.client, "persist");
expect(hoisted.setActiveMatrixClient).toHaveBeenNthCalledWith(1, hoisted.client, "default");
expect(hoisted.setActiveMatrixClient).toHaveBeenNthCalledWith(2, null, "default");
expectPersistRelease();
});
it("disables cold-start backlog dropping only when sync state is cleanly persisted", async () => {
@@ -854,24 +589,22 @@ describe("monitorMatrixProvider", () => {
expect(handlerParams.dropPreStartupMessages).toBe(false);
});
it("stops sync, drains decryptions, then waits for in-flight handlers before persisting", async () => {
it("detaches listeners, closes admission, waits for handlers, then releases", async () => {
const abortController = new AbortController();
let resolveHandler: (() => void) | null = null;
const pendingHandlers = new Map<string, () => void>();
hoisted.createMatrixRoomMessageHandler.mockReturnValue(
vi.fn(() => {
hoisted.callOrder.push("handler-start");
vi.fn((_roomId: string, event: unknown) => {
const eventId = (event as { event_id: string }).event_id;
hoisted.callOrder.push(`handler-start:${eventId}`);
return new Promise<void>((resolve) => {
resolveHandler = () => {
hoisted.callOrder.push("handler-done");
pendingHandlers.set(eventId, () => {
hoisted.callOrder.push(`handler-done:${eventId}`);
resolve();
};
});
});
}),
);
hoisted.client.stopSyncWithoutPersist.mockImplementation(() => {
hoisted.callOrder.push("pause-client");
});
hoisted.client.drainPendingDecryptions.mockImplementation(async () => {
hoisted.callOrder.push("drain-decrypts");
});
@@ -879,36 +612,38 @@ describe("monitorMatrixProvider", () => {
hoisted.callOrder.push("stop-manager");
});
hoisted.releaseSharedClientInstance.mockImplementation(async () => {
await hoisted.client.drainPendingDecryptions();
await hoisted.runRegisteredMonitorRetirement();
hoisted.callOrder.push("release-client");
return true;
});
const monitorPromise = monitorMatrixProvider({ abortSignal: abortController.signal });
await waitForCallOrderEntry("start-client");
const onRoomMessage = hoisted.registeredOnRoomMessage;
if (!onRoomMessage) {
throw new Error("expected room message handler to be registered");
}
const onRoomMessage = registeredRoomMessageHandler();
const roomMessagePromise = onRoomMessage("!room:example.org", { event_id: "$event" });
abortController.abort();
await waitForCallOrderEntry("pause-client");
await waitForCallOrderEntry("dispose-monitor-events");
expect(hoisted.callOrder).not.toContain("stop-manager");
expect(hoisted.callOrder).not.toContain("release-client");
if (resolveHandler === null) {
throw new Error("expected in-flight handler to be pending");
}
(resolveHandler as () => void)();
await onRoomMessage("!room:example.org", { event_id: "$late" });
expect(hoisted.callOrder).not.toContain("handler-start:$late");
pendingHandlers.get("$event")?.();
await roomMessagePromise;
await monitorPromise;
expect(hoisted.callOrder.indexOf("pause-client")).toBeLessThan(
hoisted.callOrder.indexOf("drain-decrypts"),
);
expect(hoisted.callOrder.indexOf("drain-decrypts")).toBeLessThan(
hoisted.callOrder.indexOf("handler-done"),
hoisted.callOrder.indexOf("dispose-auto-join"),
);
expect(hoisted.callOrder.indexOf("handler-done")).toBeLessThan(
expect(hoisted.callOrder.indexOf("dispose-auto-join")).toBeLessThan(
hoisted.callOrder.indexOf("handler-done:$event"),
);
expect(hoisted.callOrder.indexOf("dispose-monitor-events")).toBeLessThan(
hoisted.callOrder.indexOf("handler-done:$event"),
);
expect(hoisted.callOrder.indexOf("handler-done:$event")).toBeLessThan(
hoisted.callOrder.indexOf("stop-manager"),
);
expect(hoisted.callOrder.indexOf("stop-manager")).toBeLessThan(
@@ -916,6 +651,52 @@ describe("monitorMatrixProvider", () => {
);
});
it("cleans up local monitor work after a retained shared-client release", async () => {
const abortController = new AbortController();
let finishHandler: (() => void) | undefined;
hoisted.createMatrixRoomMessageHandler.mockReturnValue(
vi.fn(() => {
hoisted.callOrder.push("handler-start");
return new Promise<void>((resolve) => {
finishHandler = () => {
hoisted.callOrder.push("handler-done");
resolve();
};
});
}),
);
hoisted.stopThreadBindingManager.mockImplementation(() => {
hoisted.callOrder.push("stop-manager");
});
hoisted.releaseSharedClientInstance.mockImplementation(async () => {
await hoisted.runRegisteredMonitorRetirement();
hoisted.callOrder.push("release-retained");
});
const monitorPromise = monitorMatrixProvider({ abortSignal: abortController.signal });
await waitForCallOrderEntry("start-client");
const onRoomMessage = registeredRoomMessageHandler();
const roomMessagePromise = onRoomMessage("!room:example.org", { event_id: "$event" });
await waitForCallOrderEntry("handler-start");
abortController.abort();
await waitForCallOrderEntry("dispose-monitor-events");
expect(hoisted.client.drainPendingDecryptions).not.toHaveBeenCalled();
expect(hoisted.callOrder).not.toContain("release-retained");
finishHandler?.();
await roomMessagePromise;
await monitorPromise;
expect(hoisted.callOrder.indexOf("handler-done")).toBeLessThan(
hoisted.callOrder.indexOf("stop-manager"),
);
expect(hoisted.callOrder.indexOf("stop-manager")).toBeLessThan(
hoisted.callOrder.indexOf("release-retained"),
);
});
it("wires recent-invite promotion to fail closed when room metadata is unresolved", async () => {
await startMonitorAndAbortAfterStartup();
+76 -41
View File
@@ -26,15 +26,14 @@ import type {
} from "../../types.js";
import { resolveMatrixAccountConfig } from "../account-config.js";
import { resolveConfiguredMatrixBotUserIds } from "../accounts.js";
import { setActiveMatrixClient } from "../active-client.js";
import {
acquireSharedMatrixClient,
backfillMatrixAuthDeviceIdAfterStartup,
isBunRuntime,
resolveMatrixAuth,
resolveMatrixAuthContext,
resolveSharedMatrixClient,
type SharedMatrixClientLease,
} from "../client.js";
import { releaseSharedClientInstance } from "../client/shared.js";
import type { MatrixClient } from "../sdk.js";
import { isMatrixStartupAbortError } from "../startup-abort.js";
import {
@@ -201,36 +200,34 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
statusSink: opts.setStatus,
});
let cleanedUp = false;
let cleanupPromise: Promise<void> | null = null;
let client: MatrixClient | null = null;
let clientLease: SharedMatrixClientLease | null = null;
let monitorLifecycleSignal = opts.abortSignal;
let threadBindingManager: { accountId: string; stop: () => void } | null = null;
const monitorTaskRunner = createMatrixMonitorTaskRunner({
logger,
logVerboseMessage,
});
let disposeAutoJoin = () => {};
let disposeMonitorEvents = () => {};
let syncLifecycle: ReturnType<typeof createMatrixMonitorSyncLifecycle> | null = null;
const cleanup = async (mode: "persist" | "stop" = "persist") => {
if (cleanedUp) {
return;
let monitorSetupClosed = false;
const cleanup = (mode: "persist" | "stop" = "persist"): Promise<void> => {
if (cleanupPromise) {
return cleanupPromise;
}
cleanedUp = true;
try {
client?.stopSyncWithoutPersist();
if (client && mode === "persist") {
await client.drainPendingDecryptions("matrix monitor shutdown");
cleanupPromise = (async () => {
try {
await clientLease?.release({
mode,
});
} finally {
statusController.markStopped();
}
if (mode === "persist") {
await monitorTaskRunner.waitForIdle();
}
threadBindingManager?.stop();
if (client) {
await releaseSharedClientInstance(client, mode);
}
} finally {
client?.off("sync.state", onSyncState);
syncLifecycle?.dispose();
statusController.markStopped();
setActiveMatrixClient(null, auth.accountId);
}
})();
return cleanupPromise;
};
const defaultGroupPolicy = resolveDefaultGroupPolicy(cfg);
@@ -295,15 +292,35 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
const onSyncState = (state: MatrixSyncState) => {
noteSyncHealthState(state);
};
const monitorRetirement = {
closeTaskAdmission: () => {
monitorSetupClosed = true;
monitorTaskRunner.close();
},
detachListeners: () => {
disposeAutoJoin();
disposeMonitorEvents();
client?.off("sync.state", onSyncState);
syncLifecycle?.dispose();
},
waitForTasks: monitorTaskRunner.waitForIdle,
cleanup: () => threadBindingManager?.stop(),
};
try {
client = await resolveSharedMatrixClient({
clientLease = await acquireSharedMatrixClient({
cfg,
auth: authWithLimit,
startClient: false,
accountId: auth.accountId,
abortSignal: opts.abortSignal,
role: "monitor",
});
setActiveMatrixClient(client, auth.accountId);
client = clientLease.client;
monitorLifecycleSignal = opts.abortSignal
? AbortSignal.any([opts.abortSignal, clientLease.abortSignal])
: clientLease.abortSignal;
clientLease.registerMonitorRetirement(monitorRetirement);
const inboundDeduper = createMatrixInboundEventDeduper({
auth,
env: process.env,
@@ -311,7 +328,7 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
syncLifecycle = createMatrixMonitorSyncLifecycle({
client,
statusController,
isStopping: () => cleanedUp || opts.abortSignal?.aborted === true,
isStopping: () => cleanedUp || monitorLifecycleSignal?.aborted === true,
});
client.on("sync.state", onSyncState);
// Cold starts should ignore old room history, but once we have a persisted
@@ -372,7 +389,12 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
}
},
});
registerMatrixAutoJoin({ client, accountConfig, runtime });
disposeAutoJoin = registerMatrixAutoJoin({
client,
accountConfig,
runtime,
runDetachedTask: monitorTaskRunner.runDetachedTask,
});
const handleRoomMessage = createMatrixRoomMessageHandler({
client,
core,
@@ -411,7 +433,7 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
getMemberDisplayName,
needsRoomAliasesForConfig,
});
threadBindingManager = await createMatrixThreadBindingManager({
const createdThreadBindingManager = await createMatrixThreadBindingManager({
cfg,
accountId: effectiveAccountId,
auth,
@@ -421,11 +443,17 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
maxAgeMs: threadBindingMaxAgeMs,
logVerboseMessage,
});
if (monitorSetupClosed) {
createdThreadBindingManager.stop();
await cleanup("stop");
return;
}
threadBindingManager = createdThreadBindingManager;
logVerboseMessage(
`matrix: thread bindings ready account=${threadBindingManager.accountId} idleMs=${threadBindingIdleTimeoutMs} maxAgeMs=${threadBindingMaxAgeMs}`,
);
registerMatrixMonitorEvents({
disposeMonitorEvents = registerMatrixMonitorEvents({
cfg,
client,
auth,
@@ -456,20 +484,18 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
// Register Matrix thread bindings before the client starts syncing so threaded
// commands during startup never observe Matrix as "unavailable".
logVerboseMessage("matrix: starting client");
await resolveSharedMatrixClient({
cfg,
auth: authWithLimit,
accountId: auth.accountId,
abortSignal: opts.abortSignal,
});
await clientLease.start(monitorLifecycleSignal);
if (monitorSetupClosed) {
await cleanup("stop");
return;
}
logVerboseMessage("matrix: client started");
// Shared client is already started via resolveSharedMatrixClient.
logger.info(`matrix: logged in as ${auth.userId}`);
void backfillMatrixAuthDeviceIdAfterStartup({
auth,
env: process.env,
abortSignal: opts.abortSignal,
abortSignal: monitorLifecycleSignal,
}).catch((err: unknown) => {
logVerboseMessage(`matrix: failed to backfill deviceId after startup (${String(err)})`);
});
@@ -482,7 +508,7 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
context: {
client,
},
abortSignal: opts.abortSignal,
abortSignal: monitorLifecycleSignal,
});
await runMatrixStartupMaintenance({
@@ -502,11 +528,15 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
},
loadWebMedia: async (url, maxBytes) => await core.media.loadWebMedia(url, maxBytes),
env: process.env,
abortSignal: opts.abortSignal,
abortSignal: monitorLifecycleSignal,
});
if (monitorSetupClosed) {
await cleanup("stop");
return;
}
await Promise.race([
waitUntilAbort(opts.abortSignal, async () => {
waitUntilAbort(monitorLifecycleSignal, async () => {
try {
logVerboseMessage("matrix: stopping client");
await cleanup();
@@ -518,8 +548,13 @@ export async function monitorMatrixProvider(opts: MonitorMatrixOpts = {}): Promi
}),
syncLifecycle.waitForFatalStop(),
]);
await cleanup();
} catch (err) {
if (opts.abortSignal?.aborted === true && isMatrixStartupAbortError(err)) {
if (monitorSetupClosed) {
await cleanup("stop");
return;
}
if (monitorLifecycleSignal?.aborted === true && isMatrixStartupAbortError(err)) {
await cleanup("stop");
return;
}
@@ -6,8 +6,12 @@ export function createMatrixMonitorTaskRunner(params: {
logVerboseMessage: (message: string) => void;
}) {
const inFlight = new Set<Promise<void>>();
let closed = false;
const runDetachedTask = (label: string, task: () => Promise<void>): Promise<void> => {
if (closed) {
return Promise.resolve();
}
const trackedTask: Promise<void> = Promise.resolve()
.then(task)
.catch((error: unknown) => {
@@ -32,6 +36,7 @@ export function createMatrixMonitorTaskRunner(params: {
};
return {
close: () => (closed = true),
runDetachedTask,
waitForIdle,
};
+444 -35
View File
@@ -2,19 +2,23 @@
import "fake-indexeddb/auto";
import { EventEmitter } from "node:events";
import fs from "node:fs";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import { CryptoEvent } from "matrix-js-sdk/lib/crypto-api/CryptoEvent.js";
import type { DecryptionFailureCode as DecryptionFailureCodeValue } from "matrix-js-sdk/lib/crypto-api/index.js";
import { MatrixError } from "matrix-js-sdk/lib/http-api/errors.js";
import { MsgType } from "matrix-js-sdk/lib/matrix.js";
import { type MatrixEvent, MsgType } from "matrix-js-sdk/lib/matrix.js";
import { EventStatus } from "matrix-js-sdk/lib/models/event-status.js";
import { SyncApi, SyncState } from "matrix-js-sdk/lib/sync.js";
import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { installMatrixTestRuntime } from "../test-runtime.js";
import { readMatrixRecoveryKeyStateForPath } from "./crypto-state-store.js";
import { MatrixDecryptBridge } from "./sdk/decrypt-bridge.js";
const requireMatrixJsSdkPackage = createRequire(import.meta.url);
function requestUrl(input: RequestInfo | URL | undefined): string {
if (!input) {
return "";
@@ -114,6 +118,7 @@ class FakeMatrixEvent extends EventEmitter {
};
private decryptionFailureReasonValue: DecryptionFailureCodeValue | null;
private decryptionFailure: boolean;
private decryptionPromise: Promise<void> | null = null;
private decryptAttemptHandler?: (options?: { isRetry?: boolean }) => Promise<void> | void;
readonly attemptDecryption = vi.fn(
async (_crypto: unknown, options?: { isRetry?: boolean }): Promise<void> => {
@@ -208,6 +213,14 @@ class FakeMatrixEvent extends EventEmitter {
return this.encrypted && this.clearEvent === undefined;
}
getDecryptionPromise(): Promise<void> | null {
return this.decryptionPromise;
}
setDecryptionPromise(promise: Promise<void> | null): void {
this.decryptionPromise = promise;
}
onAttemptDecryption(handler: (options?: { isRetry?: boolean }) => Promise<void> | void): void {
this.decryptAttemptHandler = handler;
}
@@ -235,6 +248,7 @@ class FakeMatrixEvent extends EventEmitter {
}
type MatrixJsClientStub = {
classicSyncStop: ReturnType<typeof vi.fn>;
emit: (eventName: string | symbol, ...args: unknown[]) => boolean;
on: (eventName: string | symbol, listener: (...args: unknown[]) => void) => MatrixJsClientStub;
startClient: ReturnType<typeof vi.fn>;
@@ -267,10 +281,20 @@ type MatrixJsClientStub = {
getCrypto: ReturnType<typeof vi.fn<() => unknown>>;
decryptEventIfNeeded: ReturnType<typeof vi.fn>;
relations: ReturnType<typeof vi.fn>;
syncApi: SyncApi;
};
function createMatrixJsClientStub(): MatrixJsClientStub {
const client = new EventEmitter() as unknown as MatrixJsClientStub;
client.classicSyncStop = vi.fn(() => {
queueMicrotask(() => {
client.emit("sync", SyncState.Stopped, SyncState.Syncing, undefined);
});
});
client.syncApi = Object.assign(Object.create(SyncApi.prototype) as SyncApi, {
getSyncState: vi.fn(() => SyncState.Syncing),
stop: client.classicSyncStop,
});
client.startClient = vi.fn(async () => {
queueMicrotask(() => {
client.emit("sync", "PREPARED", null, undefined);
@@ -344,6 +368,10 @@ function createMatrixJsClientStub(): MatrixJsClientStub {
return client;
}
function clearMatrixSyncApiForNeverStartedClient(): void {
(matrixJsClient as { syncApi?: SyncApi }).syncApi = undefined;
}
let matrixJsClient = createMatrixJsClientStub();
let lastCreateClientOpts: Record<string, unknown> | null = null;
@@ -1284,8 +1312,9 @@ describe("MatrixClient request hardening", () => {
await assertion;
});
it("wires the sync store into the SDK and flushes it on shutdown", async () => {
it("wires the sync store into the SDK and flushes it with one SDK stop", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-sdk-store-"));
clearMatrixSyncApiForNeverStartedClient();
try {
const client = new MatrixClient("https://matrix.example.org", "token", {
@@ -1306,6 +1335,229 @@ describe("MatrixClient request hardening", () => {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
it("persists crypto before marking and flushing the clean sync cursor", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-sdk-store-"));
clearMatrixSyncApiForNeverStartedClient();
const cause = new Error("sync store flush failed");
let resolveDatabases: ((databases: IDBDatabaseInfo[]) => void) | undefined;
const pendingDatabases = new Promise<IDBDatabaseInfo[]>((resolve) => {
resolveDatabases = resolve;
});
const databasesSpy = vi.spyOn(indexedDB, "databases").mockReturnValue(pendingDatabases);
let shutdown: Promise<void> | undefined;
try {
const client = new MatrixClient("https://matrix.example.org", "token", {
storageRootDir: tempDir,
idbSnapshotPath: path.join(tempDir, "crypto-idb-snapshot.json"),
});
const store = lastCreateClientOpts?.store as
| { flush: () => Promise<void>; markCleanShutdown: () => void }
| undefined;
if (!store) {
throw new Error("expected Matrix sync store");
}
const flushSpy = vi.spyOn(store, "flush").mockRejectedValue(cause);
const markCleanSpy = vi.spyOn(store, "markCleanShutdown");
shutdown = client.stopAndPersist();
await vi.waitFor(() => {
expect(databasesSpy).toHaveBeenCalled();
});
expect(markCleanSpy).not.toHaveBeenCalled();
expect(flushSpy).not.toHaveBeenCalled();
resolveDatabases?.([]);
await expect(shutdown).rejects.toBe(cause);
expect(markCleanSpy).toHaveBeenCalledTimes(1);
expect(flushSpy).toHaveBeenCalledTimes(1);
expect(matrixJsClient.stopClient).toHaveBeenCalledTimes(1);
} finally {
resolveDatabases?.([]);
await shutdown?.catch(() => undefined);
databasesSpy.mockRestore();
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
it("does not mark or flush the sync cursor when strict crypto persistence fails", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-sdk-store-"));
clearMatrixSyncApiForNeverStartedClient();
const cause = new Error("indexeddb unavailable");
const databasesSpy = vi.spyOn(indexedDB, "databases").mockRejectedValue(cause);
try {
const client = new MatrixClient("https://matrix.example.org", "token", {
storageRootDir: tempDir,
idbSnapshotPath: path.join(tempDir, "crypto-idb-snapshot.json"),
});
const store = lastCreateClientOpts?.store as
| { flush: () => Promise<void>; markCleanShutdown: () => void }
| undefined;
if (!store) {
throw new Error("expected Matrix sync store");
}
const flushSpy = vi.spyOn(store, "flush");
const markCleanSpy = vi.spyOn(store, "markCleanShutdown");
await expect(client.stopAndPersist()).rejects.toBe(cause);
expect(markCleanSpy).not.toHaveBeenCalled();
expect(flushSpy).not.toHaveBeenCalled();
expect(matrixJsClient.stopClient).toHaveBeenCalledTimes(1);
} finally {
databasesSpy.mockRestore();
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
it("falls back to one non-persisting SDK stop when public stop persistence fails", async () => {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-sdk-stop-"));
clearMatrixSyncApiForNeverStartedClient();
const cause = new Error("indexeddb unavailable");
const databasesSpy = vi.spyOn(indexedDB, "databases").mockRejectedValue(cause);
try {
const client = new MatrixClient("https://matrix.example.org", "token", {
storageRootDir: tempDir,
idbSnapshotPath: path.join(tempDir, "crypto-idb-snapshot.json"),
});
const store = lastCreateClientOpts?.store as
| { discardPendingSyncCursorPersistence: () => void }
| undefined;
if (!store) {
throw new Error("expected Matrix sync store");
}
const discardSpy = vi.spyOn(store, "discardPendingSyncCursorPersistence");
client.stop();
await vi.waitFor(() => {
expect(discardSpy).toHaveBeenCalledTimes(1);
});
expect(matrixJsClient.stopClient).toHaveBeenCalledTimes(1);
} finally {
databasesSpy.mockRestore();
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
it("falls back to one public SDK stop when quiesce fails", async () => {
const client = new MatrixClient("https://matrix.example.org", "token");
matrixJsClient.syncApi = {} as SyncApi;
client.stop();
await vi.waitFor(() => {
expect(matrixJsClient.stopClient).toHaveBeenCalledTimes(1);
});
});
it("arms and removes the STOPPED waiter around protected classic sync stop", async () => {
const client = new MatrixClient("https://matrix.example.org", "token");
await client.start();
const emitter = (
client as unknown as {
emitter: EventEmitter;
}
).emitter;
const listenerCountBefore = emitter.listenerCount("sync.state");
const syncStop = matrixJsClient.classicSyncStop;
syncStop.mockImplementation(() => {
expect(emitter.listenerCount("sync.state")).toBe(listenerCountBefore + 1);
queueMicrotask(() => {
matrixJsClient.emit("sync", SyncState.Stopped, SyncState.Syncing, undefined);
});
});
await client.quiesceSync();
expect(syncStop).toHaveBeenCalledTimes(1);
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
expect(emitter.listenerCount("sync.state")).toBe(listenerCountBefore);
});
it("stops classic sync created by a partial startup before readiness", async () => {
const client = new MatrixClient("https://matrix.example.org", "token");
const abortController = new AbortController();
matrixJsClient.startClient.mockImplementation(() => {});
const startup = client.start({ abortSignal: abortController.signal });
await vi.waitFor(() => {
expect(matrixJsClient.startClient).toHaveBeenCalledTimes(1);
});
abortController.abort();
await expectAbortError(startup);
const syncStop = matrixJsClient.classicSyncStop;
await client.quiesceSync();
expect(syncStop).toHaveBeenCalledTimes(1);
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
});
it("times out classic sync quiesce without public stop and removes its waiter", async () => {
vi.useFakeTimers();
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-sync-timeout-"));
try {
const client = new MatrixClient("https://matrix.example.org", "token", {
storageRootDir: tempDir,
});
await client.start();
const emitter = (
client as unknown as {
emitter: EventEmitter;
}
).emitter;
const listenerCountBefore = emitter.listenerCount("sync.state");
matrixJsClient.classicSyncStop.mockImplementation(() => {});
const quiesce = client.quiesceSync();
const rejection = expect(quiesce).rejects.toThrow(
"Matrix classic sync did not reach STOPPED within 5000ms",
);
await vi.advanceTimersByTimeAsync(5_000);
await rejection;
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
expect(emitter.listenerCount("sync.state")).toBe(listenerCountBefore);
expect(vi.getTimerCount()).toBe(0);
} finally {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
it("rejects missing or non-classic sync implementations", async () => {
const client = new MatrixClient("https://matrix.example.org", "token");
await client.start();
matrixJsClient.syncApi = {} as SyncApi;
await expect(client.quiesceSync()).rejects.toThrow(
"rejected a sliding or unknown matrix-js-sdk sync implementation",
);
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
});
it("fails closed before protected sync access when the runtime SDK version drifts", async () => {
const manifest = requireMatrixJsSdkPackage("matrix-js-sdk/package.json") as {
version: string;
};
const originalVersion = manifest.version;
const syncStop = matrixJsClient.classicSyncStop;
manifest.version = "41.9.1";
try {
const client = new MatrixClient("https://matrix.example.org", "token");
await expect(client.quiesceSync()).rejects.toThrow(
"Matrix sync quiesce requires matrix-js-sdk 41.9.0; found 41.9.1",
);
expect(syncStop).not.toHaveBeenCalled();
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
} finally {
manifest.version = originalVersion;
}
});
});
describe("MatrixClient event bridge", () => {
@@ -1456,7 +1708,7 @@ describe("MatrixClient event bridge", () => {
expect(delivered).toEqual(["m.room.message"]);
});
it("can drain pending decrypt retries after sync stops", async () => {
it("quiesces and drains decrypt retries before stopping the SDK once", async () => {
vi.useFakeTimers();
const client = new MatrixClient("https://matrix.example.org", "token");
const delivered: string[] = [];
@@ -1494,12 +1746,117 @@ describe("MatrixClient event bridge", () => {
matrixJsClient.emit("event", encrypted);
encrypted.emit("decrypted", encrypted, new Error("missing room key"));
client.stopSyncWithoutPersist();
await client.drainPendingDecryptions("test shutdown");
expect(matrixJsClient.stopClient).toHaveBeenCalledTimes(1);
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
expect(matrixJsClient.decryptEventIfNeeded).toHaveBeenCalledTimes(1);
expect(delivered).toEqual(["m.room.message"]);
const lateEncrypted = new FakeMatrixEvent({
roomId: "!room:example.org",
eventId: "$late",
sender: "@alice:example.org",
type: "m.room.encrypted",
ts: Date.now(),
content: {},
decryptionFailure: true,
});
matrixJsClient.emit("event", lateEncrypted);
lateEncrypted.emit("decrypted", lateEncrypted, new Error("late missing room key"));
await Promise.resolve();
expect(matrixJsClient.decryptEventIfNeeded).toHaveBeenCalledTimes(1);
expect(delivered).toEqual(["m.room.message"]);
vi.useRealTimers();
await client.stopAndPersist();
expect(matrixJsClient.stopClient).toHaveBeenCalledTimes(1);
});
it("waits for an SDK decrypt already pending when the event is attached", async () => {
vi.useFakeTimers();
const client = new MatrixClient("https://matrix.example.org", "token");
const delivered: string[] = [];
let releaseSdkDecryption: (() => void) | undefined;
const encrypted = new FakeMatrixEvent({
roomId: "!room:example.org",
eventId: "$pending",
sender: "@alice:example.org",
type: "m.room.encrypted",
ts: Date.now(),
content: {},
});
const sdkDecryption = new Promise<void>((resolve) => {
releaseSdkDecryption = () => {
encrypted.markDecrypted({
type: "m.room.message",
content: { msgtype: "m.text", body: "decrypted before shutdown" },
});
encrypted.setDecryptionPromise(null);
encrypted.emit("decrypted", encrypted);
resolve();
};
});
encrypted.setDecryptionPromise(sdkDecryption);
client.on("room.message", (_roomId, event) => {
delivered.push(event.type);
});
await client.start();
matrixJsClient.emit("event", encrypted);
let drained = false;
const drain = client.drainPendingDecryptions("test shutdown").then(() => {
drained = true;
});
await Promise.resolve();
expect(drained).toBe(false);
expect(delivered).toHaveLength(0);
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
releaseSdkDecryption?.();
await drain;
expect(delivered).toEqual(["m.room.message"]);
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
expect(vi.getTimerCount()).toBe(0);
});
it("bounds a never-settling SDK decryption drain", async () => {
vi.useFakeTimers();
const client = new MatrixClient("https://matrix.example.org", "token");
let rejectSdkDecryption: ((reason?: unknown) => void) | undefined;
const encrypted = new FakeMatrixEvent({
roomId: "!room:example.org",
eventId: "$pending",
sender: "@alice:example.org",
type: "m.room.encrypted",
ts: Date.now(),
content: {},
});
encrypted.setDecryptionPromise(
new Promise<void>((_, reject) => {
rejectSdkDecryption = reject;
}),
);
await client.start();
matrixJsClient.emit("event", encrypted);
const drain = client.drainPendingDecryptions("test shutdown");
const rejection = expect(drain).rejects.toThrow(
"Matrix decryption drain did not finish within 5000ms",
);
await vi.advanceTimersByTimeAsync(4_999);
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
await rejection;
expect(vi.getTimerCount()).toBe(0);
rejectSdkDecryption?.(new Error("late SDK decryption failure"));
await Promise.resolve();
expect(matrixJsClient.stopClient).not.toHaveBeenCalled();
client.stopWithoutPersist();
});
it("retries failed decryptions immediately on crypto key update signals", async () => {
@@ -1800,7 +2157,7 @@ describe("MatrixClient event bridge", () => {
expect(encrypted.attemptDecryption).toHaveBeenCalledTimes(1);
expect(delivered).toEqual(["m.room.message"]);
} finally {
client.stopSyncWithoutPersist();
client.stopWithoutPersist();
}
});
@@ -1934,7 +2291,7 @@ describe("MatrixClient event bridge", () => {
await Promise.resolve();
expect(delivered).toEqual(["m.room.message"]);
} finally {
client.stopSyncWithoutPersist();
client.stopWithoutPersist();
}
});
@@ -2112,34 +2469,17 @@ describe("MatrixClient event bridge", () => {
await startExpectation;
});
it("clears stale sync state before a restarted sync session waits for fresh readiness", async () => {
matrixJsClient.startClient = vi
.fn(async () => {
queueMicrotask(() => {
matrixJsClient.emit("sync", "PREPARED", null, undefined);
});
})
.mockImplementationOnce(async () => {
queueMicrotask(() => {
matrixJsClient.emit("sync", "PREPARED", null, undefined);
});
})
.mockImplementationOnce(async () => {});
it("rejects restarting a fully stopped client and requires a new shared generation", async () => {
const client = new MatrixClient("https://matrix.example.org", "token");
await client.start();
client.stopSyncWithoutPersist();
client.stopWithoutPersist();
vi.useFakeTimers();
const restartPromise = client.start();
const restartExpectation = expect(restartPromise).rejects.toThrow(
"Matrix client did not reach a ready sync state within 30000ms",
await expect(client.start()).rejects.toThrow(
"Matrix client has been fully stopped and cannot be restarted; acquire a new shared client generation",
);
await vi.advanceTimersByTimeAsync(30_000);
await restartExpectation;
expect(matrixJsClient.startClient).toHaveBeenCalledTimes(1);
expect(matrixJsClient.stopClient).toHaveBeenCalledTimes(1);
});
it("replays outstanding invite rooms at startup", async () => {
@@ -4426,7 +4766,7 @@ describe("MatrixClient crypto bootstrapping", () => {
expect(event.attemptDecryption).toHaveBeenCalledTimes(1);
});
it("does not rearm or emit after stop while a decrypt retry is in flight", async () => {
it("lets an in-flight retry finish during quiesce without rearming it", async () => {
let releaseDecrypt: (() => void) | undefined;
const emitFailedDecryption = vi.fn();
const emitMessage = vi.fn();
@@ -4473,12 +4813,10 @@ describe("MatrixClient crypto bootstrapping", () => {
}
).decryptRetries.set("!room:example.org|$event", retryState);
bridge.retryPendingNow("test retry");
const drain = bridge.drainPendingDecryptions("test shutdown");
await Promise.resolve();
bridge.stop();
releaseDecrypt?.();
await Promise.resolve();
await Promise.resolve();
await drain;
expect(emitFailedDecryption).not.toHaveBeenCalled();
expect(emitMessage).not.toHaveBeenCalled();
@@ -4490,5 +4828,76 @@ describe("MatrixClient crypto bootstrapping", () => {
).decryptRetries.size,
).toBe(0);
});
it("blocks new encrypted events and crypto retry resurrection after quiescing", async () => {
const listeners = new Map<string, () => void>();
const emitMessage = vi.fn();
const cryptoApi = {
on: (eventName: string, listener: () => void) => {
listeners.set(eventName, listener);
},
};
const bridge = new MatrixDecryptBridge({
client: {
getCrypto: () => cryptoApi,
},
toRaw: (event) => ({ event_id: event.getId() ?? "" }),
emitDecryptedEvent: vi.fn(),
emitFailedDecryption: vi.fn(),
emitMessage,
});
const exhausted = new FakeMatrixEvent({
roomId: "!room:example.org",
eventId: "$exhausted",
sender: "@alice:example.org",
type: "m.room.encrypted",
ts: Date.now(),
content: {},
decryptionFailure: true,
});
(
bridge as unknown as {
exhaustedDecryptRetries: Map<
string,
{
event: FakeMatrixEvent;
roomId: string;
eventId: string;
attempts: number;
inFlight: boolean;
timer: ReturnType<typeof setTimeout> | null;
exhaustedAt: number;
}
>;
}
).exhaustedDecryptRetries.set("!room:example.org|$exhausted", {
event: exhausted,
roomId: "!room:example.org",
eventId: "$exhausted",
attempts: 8,
inFlight: false,
timer: null,
exhaustedAt: Date.now(),
});
bridge.bindCryptoRetrySignals(cryptoApi);
await bridge.drainPendingDecryptions("test shutdown");
listeners.get(CryptoEvent.KeyBackupDecryptionKeyCached)?.();
const late = new FakeMatrixEvent({
roomId: "!room:example.org",
eventId: "$late",
sender: "@alice:example.org",
type: "m.room.encrypted",
ts: Date.now(),
content: {},
});
bridge.attachEncryptedEvent(late as unknown as MatrixEvent, "!room:example.org");
late.emit("decrypted", late);
await Promise.resolve();
expect(exhausted.attemptDecryption).not.toHaveBeenCalled();
expect(emitMessage).not.toHaveBeenCalled();
});
});
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
+46 -30
View File
@@ -27,6 +27,7 @@ import {
type MatrixOwnDeviceInfo,
type MatrixOwnDeviceVerificationStatus,
} from "./client-support.js";
import { quiesceMatrixClientSync } from "./client-sync-quiesce.js";
import type { MatrixCryptoFacade } from "./crypto-facade.js";
import type { MatrixDecryptBridge } from "./decrypt-bridge.js";
import { matrixEventToRaw } from "./event-helpers.js";
@@ -145,6 +146,7 @@ export abstract class MatrixClientBase {
protected transactionScopeId: string | null = null;
protected transactionScopePromise: Promise<string> | null = null;
private readonly messageWireDispatchGuards = new Map<string, MatrixMessageWireDispatchGuard>();
private sdkStopped = false;
readonly dms = {
update: async (): Promise<boolean> => {
@@ -480,6 +482,11 @@ export abstract class MatrixClientBase {
if (this.started) {
return;
}
if (this.sdkStopped) {
throw new Error(
"Matrix client has been fully stopped and cannot be restarted; acquire a new shared client generation",
);
}
throwIfMatrixStartupAborted(opts.abortSignal);
await this.ensureCryptoSupportInitialized();
@@ -535,7 +542,10 @@ export abstract class MatrixClientBase {
await this.startSyncSession({ bootstrapCrypto: false });
}
stopSyncWithoutPersist(): void {
private stopSdkClient(): void {
if (this.sdkStopped) {
return;
}
if (this.idbPersistTimer) {
clearInterval(this.idbPersistTimer);
this.idbPersistTimer = null;
@@ -543,50 +553,56 @@ export abstract class MatrixClientBase {
this.currentSyncState = null;
this.currentSyncError = undefined;
this.client.stopClient();
this.sdkStopped = true;
this.started = false;
}
async quiesceSync(): Promise<void> {
await quiesceMatrixClientSync({
client: this.client,
emitter: this.emitter,
markStopped: () => {
this.started = false;
},
started: this.started,
syncStore: this.syncStore,
});
}
async drainPendingDecryptions(reason = "matrix client shutdown"): Promise<void> {
await this.decryptBridge?.drainPendingDecryptions(reason);
}
stop(): void {
this.stopSyncWithoutPersist();
this.decryptBridge?.stop();
// Final persist on shutdown
this.syncStore?.markCleanShutdown();
if (loadedMatrixCryptoRuntime) {
const { persistIdbToDisk } = loadedMatrixCryptoRuntime;
this.stopPersistPromise = Promise.all([
persistIdbToDisk({
snapshotPath: this.idbSnapshotPath,
databasePrefix: this.cryptoDatabasePrefix,
}).catch(noop),
this.syncStore?.flush().catch(noop),
]).then(() => undefined);
return;
}
this.stopPersistPromise = loadMatrixCryptoRuntime()
.then(async ({ persistIdbToDisk }) => {
await Promise.all([
persistIdbToDisk({
snapshotPath: this.idbSnapshotPath,
databasePrefix: this.cryptoDatabasePrefix,
}).catch(noop),
this.syncStore?.flush().catch(noop),
]);
})
.catch(noop)
.then(() => undefined);
void this.stopAndPersist()
.catch(() => this.stopWithoutPersist())
.catch(noop);
}
async stopAndPersist(): Promise<void> {
this.stop();
if (this.stopPersistPromise) {
await this.stopPersistPromise;
return;
}
this.stopPersistPromise = (async () => {
await this.quiesceSync();
this.stopSdkClient();
this.decryptBridge?.stop();
const runtime = loadedMatrixCryptoRuntime ?? (await loadMatrixCryptoRuntime());
await runtime.persistIdbToDisk({
snapshotPath: this.idbSnapshotPath,
databasePrefix: this.cryptoDatabasePrefix,
strict: true,
});
this.syncStore?.markCleanShutdown();
await this.syncStore?.flush();
})();
await this.stopPersistPromise;
}
stopWithoutPersist(): void {
this.stopSyncWithoutPersist();
this.syncStore?.discardPendingSyncCursorPersistence();
this.stopSdkClient();
this.decryptBridge?.stop();
this.stopPersistPromise = Promise.resolve();
}
@@ -0,0 +1,94 @@
import { EventEmitter } from "node:events";
import { createRequire } from "node:module";
import type { MatrixClient as MatrixJsClient } from "matrix-js-sdk/lib/matrix.js";
import { SyncApi, SyncState } from "matrix-js-sdk/lib/sync.js";
import type { SqliteBackedMatrixSyncStore } from "../client/file-sync-store.js";
import type { MatrixSyncState } from "../sync-state.js";
const MATRIX_SYNC_QUIESCE_TIMEOUT_MS = 5_000;
const MATRIX_JS_SDK_SYNC_VERSION = "41.9.0";
const matrixJsSdkPackage = createRequire(import.meta.url)("matrix-js-sdk/package.json") as {
version?: unknown;
};
function assertMatrixJsSdkSyncVersion(): void {
const version = matrixJsSdkPackage.version;
if (version !== MATRIX_JS_SDK_SYNC_VERSION) {
throw new Error(
`Matrix sync quiesce requires matrix-js-sdk ${MATRIX_JS_SDK_SYNC_VERSION}; found ${String(version)}`,
);
}
}
export async function quiesceMatrixClientSync(params: {
client: MatrixJsClient;
emitter: EventEmitter;
markStopped: () => void;
started: boolean;
syncStore?: Pick<
SqliteBackedMatrixSyncStore,
"discardPendingSyncCursorPersistence" | "freezeSyncCursorPersistence"
>;
}): Promise<void> {
await params.syncStore?.freezeSyncCursorPersistence();
try {
assertMatrixJsSdkSyncVersion();
} catch (error) {
params.syncStore?.discardPendingSyncCursorPersistence();
throw error;
}
// 41.9.0: stop protected classic sync here; public stopClient also stops crypto.
const syncApi = (params.client as MatrixJsClient & { syncApi?: unknown }).syncApi;
if (syncApi === undefined && !params.started) {
return;
}
if (!(syncApi instanceof SyncApi)) {
params.syncStore?.discardPendingSyncCursorPersistence();
throw new Error(
syncApi === undefined
? "Matrix sync quiesce requires the classic matrix-js-sdk SyncApi, but none is active"
: "Matrix sync quiesce rejected a sliding or unknown matrix-js-sdk sync implementation",
);
}
if (syncApi.getSyncState() === SyncState.Stopped) {
params.markStopped();
return;
}
await new Promise<void>((resolve, reject) => {
let settled = false;
const settle = (error?: Error) => {
if (settled) {
return;
}
settled = true;
clearTimeout(timeout);
params.emitter.off("sync.state", onSyncState);
if (error) {
reject(error);
} else {
params.markStopped();
resolve();
}
};
const onSyncState = (state: MatrixSyncState) => {
if (state === "STOPPED") {
settle();
}
};
const timeout = setTimeout(() => {
params.syncStore?.discardPendingSyncCursorPersistence();
settle(new Error(`Matrix classic sync did not reach STOPPED within 5000ms`));
}, MATRIX_SYNC_QUIESCE_TIMEOUT_MS);
timeout.unref?.();
params.emitter.on("sync.state", onSyncState);
try {
syncApi.stop();
} catch (error) {
params.syncStore?.discardPendingSyncCursorPersistence();
settle(error instanceof Error ? error : new Error(String(error)));
}
});
}
@@ -47,6 +47,7 @@ type MatrixCryptoRetrySignalSource = {
const MATRIX_DECRYPT_RETRY_BASE_DELAY_MS = 1_500;
const MATRIX_DECRYPT_RETRY_MAX_DELAY_MS = 30_000;
const MATRIX_DECRYPT_RETRY_MAX_ATTEMPTS = 8;
const MATRIX_DECRYPT_DRAIN_TIMEOUT_MS = 5_000;
const MATRIX_DECRYPT_EXHAUSTED_RETRY_TTL_MS = 60 * 60_000;
const MATRIX_DECRYPT_EXHAUSTED_RETRY_MAX_ENTRIES = 512;
@@ -71,6 +72,7 @@ function shouldRetryDecryptionFailure(event: MatrixEvent): boolean {
export class MatrixDecryptBridge<TRawEvent extends DecryptBridgeRawEvent> {
private readonly trackedEncryptedEvents = new WeakSet<object>();
private readonly pendingSdkDecryptions = new Set<Promise<void>>();
private readonly decryptedMessageDedupe = new Map<string, number>();
private readonly decryptRetries = new Map<string, MatrixDecryptRetryState>();
private readonly failedDecryptionsNotified = new Set<string>();
@@ -78,6 +80,7 @@ export class MatrixDecryptBridge<TRawEvent extends DecryptBridgeRawEvent> {
private activeRetryRuns = 0;
private readonly retryIdleResolvers = new Set<() => void>();
private cryptoRetrySignalsBound = false;
private quiescing = false;
private stopped = false;
constructor(
@@ -104,13 +107,21 @@ export class MatrixDecryptBridge<TRawEvent extends DecryptBridgeRawEvent> {
}
attachEncryptedEvent(event: MatrixEvent, roomId: string): void {
if (this.stopped) {
if (this.quiescing || this.stopped) {
return;
}
if (this.trackedEncryptedEvents.has(event)) {
return;
}
this.trackedEncryptedEvents.add(event);
const sdkDecryption = event.getDecryptionPromise();
if (sdkDecryption) {
this.pendingSdkDecryptions.add(sdkDecryption);
const forgetSdkDecryption = () => {
this.pendingSdkDecryptions.delete(sdkDecryption);
};
void sdkDecryption.then(forgetSdkDecryption, forgetSdkDecryption);
}
event.on(MatrixEventEvent.Decrypted, (decryptedEvent: MatrixEvent, err?: Error) => {
this.handleEncryptedEventDecrypted({
roomId,
@@ -127,7 +138,7 @@ export class MatrixDecryptBridge<TRawEvent extends DecryptBridgeRawEvent> {
}
retryPendingNow(reason: string, options?: { includeExhausted?: boolean }): void {
if (this.stopped) {
if (this.quiescing || this.stopped) {
return;
}
if (options?.includeExhausted) {
@@ -188,25 +199,49 @@ export class MatrixDecryptBridge<TRawEvent extends DecryptBridgeRawEvent> {
}
stop(): void {
this.quiescing = true;
this.stopped = true;
for (const retryKey of this.decryptRetries.keys()) {
this.clearDecryptRetry(retryKey);
}
this.pendingSdkDecryptions.clear();
this.exhaustedDecryptRetries.clear();
}
async drainPendingDecryptions(reason: string): Promise<void> {
for (let attempts = 0; attempts < MATRIX_DECRYPT_RETRY_MAX_ATTEMPTS; attempts += 1) {
if (this.decryptRetries.size === 0) {
return;
async drainPendingDecryptions(_reason: string): Promise<void> {
this.quiescing = true;
const pendingSdkDecryptions = Array.from(this.pendingSdkDecryptions);
const pending = Array.from(this.decryptRetries.entries());
for (const [retryKey, state] of pending) {
if (state.timer) {
clearTimeout(state.timer);
state.timer = null;
}
this.retryPendingNow(reason);
await this.waitForActiveRetryRunsToFinish();
const hasPendingRetryTimers = Array.from(this.decryptRetries.values()).some(
(state) => state.timer || state.inFlight,
);
if (!hasPendingRetryTimers) {
return;
if (!state.inFlight) {
this.runDecryptRetry(retryKey).catch(noop);
}
}
let timeout: ReturnType<typeof setTimeout> | undefined;
try {
await Promise.race([
Promise.all([
Promise.allSettled(pendingSdkDecryptions),
this.waitForActiveRetryRunsToFinish(),
]),
new Promise<never>((_, reject) => {
timeout = setTimeout(() => {
reject(
new Error(
`Matrix decryption drain did not finish within ${MATRIX_DECRYPT_DRAIN_TIMEOUT_MS}ms`,
),
);
}, MATRIX_DECRYPT_DRAIN_TIMEOUT_MS);
timeout.unref?.();
}),
]);
} finally {
if (timeout !== undefined) {
clearTimeout(timeout);
}
}
}
@@ -286,7 +321,7 @@ export class MatrixDecryptBridge<TRawEvent extends DecryptBridgeRawEvent> {
roomId: string;
eventId: string;
}): void {
if (this.stopped) {
if (this.quiescing || this.stopped) {
return;
}
const retryKey = resolveDecryptRetryKey(params.roomId, params.eventId);
@@ -387,7 +422,8 @@ export class MatrixDecryptBridge<TRawEvent extends DecryptBridgeRawEvent> {
if (this.decryptRetries.get(retryKey) !== state) {
return;
}
if (this.stopped) {
if (this.stopped || (this.quiescing && state.event.isDecryptionFailure())) {
this.clearDecryptRetry(retryKey);
return;
}
if (state.event.isDecryptionFailure()) {
@@ -166,4 +166,21 @@ describe("Matrix IndexedDB persistence", () => {
}),
).resolves.toEqual([{ key: "room-1", value: { session: "abc123" } }]);
});
it("strictly propagates final IndexedDB persistence failures", async () => {
const cause = new Error("indexeddb unavailable");
const databasesSpy = vi.spyOn(indexedDB, "databases").mockRejectedValue(cause);
try {
await expect(
persistIdbToDisk({
snapshotPath: path.join(tmpDir, "crypto-idb-snapshot.json"),
databasePrefix: DATABASE_PREFIX,
strict: true,
}),
).rejects.toBe(cause);
} finally {
databasesSpy.mockRestore();
}
});
});
@@ -306,6 +306,7 @@ export async function persistIdbToDisk(params?: {
// Production callers pass MatrixStoragePaths.idbSnapshotPath; explicit paths only isolate tests.
snapshotPath?: string;
databasePrefix?: string;
strict?: boolean;
}): Promise<void> {
const snapshotPath = params?.snapshotPath ?? resolveDefaultIdbSnapshotPath();
let callbackStarted = false;
@@ -355,6 +356,9 @@ export async function persistIdbToDisk(params?: {
throwLegacySnapshotMigrationRequired();
}
LogService.warn("IdbPersistence", "Failed to persist IndexedDB snapshot:", err);
if (params?.strict) {
throw err;
}
}
}
@@ -6,33 +6,25 @@ import {
expectOneOffSharedMatrixClient,
matrixClientResolverMocks,
primeMatrixClientResolverMocks,
setAcquiredMatrixClient,
} from "../client-resolver.test-helpers.js";
const {
getMatrixRuntimeMock,
getActiveMatrixClientMock,
acquireSharedMatrixClientMock,
releaseSharedClientInstanceMock,
sharedLeaseReleaseMock,
isBunRuntimeMock,
resolveMatrixAuthContextMock,
} = matrixClientResolverMocks;
const TEST_CFG = {};
vi.mock("../active-client.js", () => ({
getActiveMatrixClient: (...args: unknown[]) => getActiveMatrixClientMock(...args),
}));
vi.mock("../client.js", () => ({
acquireSharedMatrixClient: (...args: unknown[]) => acquireSharedMatrixClientMock(...args),
isBunRuntime: () => isBunRuntimeMock(),
resolveMatrixAuthContext: resolveMatrixAuthContextMock,
}));
vi.mock("../client/shared.js", () => ({
releaseSharedClientInstance: (...args: unknown[]) => releaseSharedClientInstanceMock(...args),
}));
vi.mock("../../runtime.js", () => ({
getMatrixRuntime: () => getMatrixRuntimeMock(),
}));
@@ -47,18 +39,14 @@ describe("matrix send client helpers", () => {
});
beforeEach(() => {
primeMatrixClientResolverMocks({
resolved: {},
});
primeMatrixClientResolverMocks({ resolved: {} });
});
afterEach(() => {
vi.unstubAllEnvs();
});
it("stops one-off shared clients when no active monitor client is registered", async () => {
vi.stubEnv("OPENCLAW_GATEWAY_PORT", "18799");
it("starts and persists borrowed send clients", async () => {
const result = await withResolvedMatrixSendClient(
{ cfg: TEST_CFG, accountId: "default" },
async () => "ok",
@@ -72,32 +60,26 @@ describe("matrix send client helpers", () => {
expect(result).toBe("ok");
});
it("reuses active monitor client when available", async () => {
const activeClient = createMockMatrixClient();
getActiveMatrixClientMock.mockReturnValue(activeClient);
it("forwards the transient retirement signal to send work", async () => {
const sharedClient = createMockMatrixClient();
const lease = setAcquiredMatrixClient(sharedClient);
const result = await withResolvedMatrixSendClient(
await withResolvedMatrixSendClient(
{ cfg: TEST_CFG, accountId: "default" },
async (client) => {
expect(client).toBe(activeClient);
return "ok";
async (_client, abortSignal) => {
expect(abortSignal).toBe(lease.abortSignal);
},
);
expect(result).toBe("ok");
expect(acquireSharedMatrixClientMock).not.toHaveBeenCalled();
expect(activeClient["start"]).toHaveBeenCalledTimes(1);
expect(activeClient["stop"]).not.toHaveBeenCalled();
expect(activeClient["stopAndPersist"]).not.toHaveBeenCalled();
});
it("uses the effective account id when auth resolution is implicit", async () => {
resolveMatrixAuthContextMock.mockReturnValue({
cfg: {},
cfg: TEST_CFG,
env: process.env,
accountId: "ops",
resolved: {},
});
await withResolvedMatrixSendClient({ cfg: TEST_CFG }, async () => {});
await expectOneOffSharedMatrixClient({
@@ -125,9 +107,9 @@ describe("matrix send client helpers", () => {
});
});
it("stops shared matrix clients when wrapped sends fail", async () => {
it("persists borrowed send clients when wrapped sends fail", async () => {
const sharedClient = createMockMatrixClient();
acquireSharedMatrixClientMock.mockResolvedValue(sharedClient);
setAcquiredMatrixClient(sharedClient);
await expect(
withResolvedMatrixSendClient({ cfg: TEST_CFG, accountId: "default" }, async () => {
@@ -135,20 +117,10 @@ describe("matrix send client helpers", () => {
}),
).rejects.toThrow("boom");
expect(releaseSharedClientInstanceMock).toHaveBeenCalledWith(sharedClient, "persist");
expect(sharedLeaseReleaseMock).toHaveBeenCalledWith({ mode: "persist" });
});
it("starts one-off clients before outbound sends so encrypted rooms can reuse live crypto state", async () => {
const sharedClient = createMockMatrixClient();
acquireSharedMatrixClientMock.mockResolvedValue(sharedClient);
await withResolvedMatrixSendClient({ cfg: TEST_CFG, accountId: "default" }, async () => "ok");
expect(sharedClient["start"]).toHaveBeenCalledTimes(1);
expect(sharedClient["prepareForOneOff"]).not.toHaveBeenCalled();
});
it("keeps one-off control clients lightweight when no active monitor client is registered", async () => {
it("keeps borrowed control clients unstarted and releases without persistence", async () => {
const result = await withResolvedMatrixControlClient(
{ cfg: TEST_CFG, accountId: "default" },
async () => "ok",
@@ -162,22 +134,19 @@ describe("matrix send client helpers", () => {
expect(result).toBe("ok");
});
it("reuses active monitor clients for control operations without restarting them", async () => {
const activeClient = createMockMatrixClient();
getActiveMatrixClientMock.mockReturnValue(activeClient);
it("does not borrow or stop explicitly injected clients", async () => {
const start = vi.fn(async () => undefined);
const injected = Object.assign(createMockMatrixClient(), { start });
const result = await withResolvedMatrixControlClient(
{ cfg: TEST_CFG, accountId: "default" },
async (client) => {
expect(client).toBe(activeClient);
return "ok";
},
);
await withResolvedMatrixSendClient({ client: injected }, async (client) => {
expect(client).toBe(injected);
});
await withResolvedMatrixControlClient({ client: injected }, async (client) => {
expect(client).toBe(injected);
});
expect(result).toBe("ok");
expect(start).not.toHaveBeenCalled();
expect(acquireSharedMatrixClientMock).not.toHaveBeenCalled();
expect(activeClient["start"]).not.toHaveBeenCalled();
expect(activeClient["stop"]).not.toHaveBeenCalled();
expect(activeClient["stopAndPersist"]).not.toHaveBeenCalled();
expect(sharedLeaseReleaseMock).not.toHaveBeenCalled();
});
});
+3 -3
View File
@@ -32,7 +32,7 @@ export async function withResolvedMatrixSendClient<T>(
timeoutMs?: number;
accountId?: string | null;
},
run: (client: MatrixClient) => Promise<T>,
run: (client: MatrixClient, abortSignal?: AbortSignal) => Promise<T>,
): Promise<T> {
return await withResolvedMatrixClient(
{
@@ -55,7 +55,7 @@ export async function withResolvedMatrixControlClient<T>(
timeoutMs?: number;
accountId?: string | null;
},
run: (client: MatrixClient) => Promise<T>,
run: (client: MatrixClient, abortSignal?: AbortSignal) => Promise<T>,
): Promise<T> {
return await withResolvedMatrixClient(
{
@@ -74,7 +74,7 @@ async function withResolvedMatrixClient<T>(
accountId?: string | null;
readiness: "started" | "none";
},
run: (client: MatrixClient) => Promise<T>,
run: (client: MatrixClient, abortSignal?: AbortSignal) => Promise<T>,
shutdownBehavior?: "persist",
): Promise<T> {
if (opts.client) {
@@ -10,29 +10,23 @@ export const MATRIX_QA_E2EE_SYNC_FILTER = {
},
};
export async function runMatrixQaE2eeClientOperation<T>(params: {
label: string;
run: () => Promise<T>;
stop: () => void;
timeoutMs: number;
}): Promise<T> {
async function withMatrixQaE2eeTimeout<T>(
promise: Promise<T>,
timeoutMs: number,
message: string,
onTimeout?: () => void,
): Promise<T> {
let timer: NodeJS.Timeout | undefined;
const timeout = new Promise<never>((_resolve, reject) => {
timer = setTimeout(() => {
// Matrix SDK encryption can wait indefinitely after room-key sharing. Stop this
// disposable QA client so the scenario can fail and release its worker resources.
try {
params.stop();
} catch {
// Preserve the operation timeout as the actionable failure.
}
reject(new Error(`${params.label} timed out after ${params.timeoutMs}ms`));
}, params.timeoutMs);
onTimeout?.();
reject(new Error(message));
}, timeoutMs);
timer.unref();
});
try {
return await Promise.race([params.run(), timeout]);
return await Promise.race([promise, timeout]);
} finally {
if (timer) {
clearTimeout(timer);
@@ -40,6 +34,83 @@ export async function runMatrixQaE2eeClientOperation<T>(params: {
}
}
export function createMatrixQaE2eeClientLifecycle(params: {
detachListeners: () => void;
drainPendingDecryptions: () => Promise<void>;
shutdownTimeoutMs: number;
stopAndPersist: () => Promise<void>;
stopWithoutPersist: () => void;
}) {
const activeOperations = new Set<Promise<unknown>>();
let shutdownStarted = false;
let stopPromise: Promise<void> | undefined;
const failShutdown = (phase: string, cause: unknown): never => {
try {
params.stopWithoutPersist();
} catch {
// Preserve the lifecycle failure that explains why persistence was skipped.
}
throw new Error(
`Matrix E2EE client shutdown failed while ${phase}; crypto state was discarded. Retry the QA scenario with a fresh client.`,
{ cause },
);
};
const stop = (): Promise<void> => {
if (stopPromise) {
return stopPromise;
}
shutdownStarted = true;
stopPromise = (async () => {
const deadline = Date.now() + params.shutdownTimeoutMs;
params.detachListeners();
if (activeOperations.size > 0) {
const graceMs = Math.min(1_000, Math.max(0, deadline - Date.now()));
await withMatrixQaE2eeTimeout(
Promise.allSettled(activeOperations),
graceMs,
"active Matrix SDK operations did not settle before shutdown",
).catch((error: unknown) => {
failShutdown("waiting for active Matrix SDK operations", error);
});
}
await withMatrixQaE2eeTimeout(
params.drainPendingDecryptions(),
Math.max(0, deadline - Date.now()),
"pending Matrix decryptions did not drain before shutdown",
).catch((error: unknown) => {
failShutdown("draining pending Matrix decryptions", error);
});
await params.stopAndPersist();
})();
return stopPromise;
};
const runMatrixQaE2eeClientOperation = async <T>(operation: {
label: string;
run: () => Promise<T>;
timeoutMs: number;
}): Promise<T> => {
if (shutdownStarted) {
throw new Error(
`Matrix E2EE client shutdown has started; cannot start ${operation.label}. Retry the QA scenario with a fresh client.`,
);
}
const active = operation.run();
activeOperations.add(active);
void active.finally(() => activeOperations.delete(active)).catch(() => undefined);
return withMatrixQaE2eeTimeout(
active,
operation.timeoutMs,
`${operation.label} timed out after ${operation.timeoutMs}ms`,
() => void stop().catch(() => undefined),
);
};
return { runOperation: runMatrixQaE2eeClientOperation, stop };
}
function shouldRecordMatrixQaObservedEventUpdate(params: {
next: MatrixQaObservedEvent;
previous: MatrixQaObservedEvent | undefined;
@@ -5,42 +5,204 @@ import path from "node:path";
import { describe, expect, it, vi } from "vitest";
import {
MATRIX_QA_E2EE_SYNC_FILTER,
createMatrixQaE2eeClientLifecycle,
createMatrixQaE2eeObservedEventRecorder,
prepareMatrixQaE2eeStorage,
runMatrixQaE2eeClientOperation,
} from "./e2ee-client-internals.js";
import { findMatrixQaObservedEventMatch, type MatrixQaObservedEvent } from "./events.js";
const testing = {
MATRIX_QA_E2EE_SYNC_FILTER,
createMatrixQaE2eeClientLifecycle,
createMatrixQaE2eeObservedEventRecorder,
findMatrixQaObservedEventMatch,
prepareMatrixQaE2eeStorage,
runMatrixQaE2eeClientOperation,
};
describe("matrix qa e2ee client storage", () => {
it("stops a disposable client when an E2EE operation exceeds its scenario timeout", async () => {
function createLifecycleFixture(options?: {
drain?: () => Promise<void>;
shutdownTimeoutMs?: number;
}) {
const calls: string[] = [];
const lifecycle = testing.createMatrixQaE2eeClientLifecycle({
detachListeners: vi.fn(() => calls.push("detach")),
drainPendingDecryptions: vi.fn(async () => {
calls.push("drain");
await options?.drain?.();
}),
shutdownTimeoutMs: options?.shutdownTimeoutMs ?? 500,
stopAndPersist: vi.fn(async () => {
calls.push("stop-and-persist");
}),
stopWithoutPersist: vi.fn(() => calls.push("stop-and-discard")),
});
return { calls, lifecycle };
}
it("drains decryptions before stopping the SDK and persisting", async () => {
const { calls, lifecycle } = createLifecycleFixture();
await lifecycle.stop();
expect(calls).toEqual(["detach", "drain", "stop-and-persist"]);
});
it("shares one stop promise across concurrent and repeated shutdown requests", async () => {
const { calls, lifecycle } = createLifecycleFixture();
const first = lifecycle.stop();
const second = lifecycle.stop();
await Promise.all([first, second]);
const third = lifecycle.stop();
const run = vi.fn(async () => "sent");
expect(second).toBe(first);
expect(third).toBe(first);
await expect(
lifecycle.runOperation({
label: "Matrix E2EE text send",
run,
timeoutMs: 100,
}),
).rejects.toThrow("shutdown has started");
expect(run).not.toHaveBeenCalled();
expect(calls).toEqual(["detach", "drain", "stop-and-persist"]);
});
it("gives an active operation a bounded grace period before draining and stopping", async () => {
vi.useFakeTimers();
try {
const stop = vi.fn();
const operation = testing.runMatrixQaE2eeClientOperation({
const { calls, lifecycle } = createLifecycleFixture();
let finishOperation: ((value: string) => void) | undefined;
const operation = lifecycle.runOperation({
label: "Matrix E2EE text send",
run: () =>
new Promise<string>((resolve) => {
calls.push("operation");
finishOperation = resolve;
}),
timeoutMs: 1_000,
});
const stop = lifecycle.stop();
expect(calls).toEqual(["operation", "detach"]);
finishOperation?.("sent");
await operation;
await stop;
expect(calls).toEqual(["operation", "detach", "drain", "stop-and-persist"]);
} finally {
vi.useRealTimers();
}
});
it("discards without persisting when active operation grace expires", async () => {
vi.useFakeTimers();
try {
const { calls, lifecycle } = createLifecycleFixture({
shutdownTimeoutMs: 100,
});
void lifecycle.runOperation({
label: "Matrix E2EE text send",
run: () =>
new Promise<string>(() => {
// Intentionally pending so the timeout owns settlement.
calls.push("operation");
}),
stop,
timeoutMs: 150_000,
timeoutMs: 1_000,
});
const rejection = expect(operation).rejects.toThrow(
"Matrix E2EE text send timed out after 150000ms",
const stop = lifecycle.stop();
const rejection = expect(stop).rejects.toThrow(
"shutdown failed while waiting for active Matrix SDK operations",
);
await vi.advanceTimersByTimeAsync(150_000);
await vi.advanceTimersByTimeAsync(100);
await rejection;
expect(stop).toHaveBeenCalledOnce();
expect(calls).toEqual(["operation", "detach", "stop-and-discard"]);
} finally {
vi.useRealTimers();
}
});
it("discards without persisting when pending decryptions exceed the shutdown deadline", async () => {
vi.useFakeTimers();
try {
const { calls, lifecycle } = createLifecycleFixture({
drain: () =>
new Promise<void>(() => {
// Intentionally pending so the shutdown deadline owns settlement.
}),
shutdownTimeoutMs: 100,
});
const stop = lifecycle.stop();
const rejection = expect(stop).rejects.toThrow(
"shutdown failed while draining pending Matrix decryptions",
);
await vi.advanceTimersByTimeAsync(100);
await rejection;
expect(calls).toEqual(["detach", "drain", "stop-and-discard"]);
} finally {
vi.useRealTimers();
}
});
it("requests lifecycle shutdown on operation timeout instead of directly discarding", async () => {
vi.useFakeTimers();
try {
const { calls, lifecycle } = createLifecycleFixture({
shutdownTimeoutMs: 100,
});
const operation = lifecycle.runOperation({
label: "Matrix E2EE text send",
run: () =>
new Promise<string>(() => {
calls.push("operation");
}),
timeoutMs: 50,
});
const rejection = expect(operation).rejects.toThrow(
"Matrix E2EE text send timed out after 50ms",
);
await vi.advanceTimersByTimeAsync(50);
await rejection;
expect(calls).toEqual(["operation", "detach"]);
await vi.advanceTimersByTimeAsync(100);
expect(calls).toEqual(["operation", "detach", "stop-and-discard"]);
} finally {
vi.useRealTimers();
}
});
it("observes a tracked operation that rejects after shutdown has discarded state", async () => {
vi.useFakeTimers();
try {
const { lifecycle } = createLifecycleFixture({
shutdownTimeoutMs: 50,
});
let rejectOperation: ((error: Error) => void) | undefined;
const operation = lifecycle.runOperation({
label: "Matrix E2EE text send",
run: () =>
new Promise<string>((_resolve, reject) => {
rejectOperation = reject;
}),
timeoutMs: 1_000,
});
const operationRejection = expect(operation).rejects.toThrow("late send failure");
const stop = lifecycle.stop();
const stopRejection = expect(stop).rejects.toThrow(
"shutdown failed while waiting for active Matrix SDK operations",
);
await vi.advanceTimersByTimeAsync(50);
await stopRejection;
rejectOperation?.(new Error("late send failure"));
await operationRejection;
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.useRealTimers();
@@ -25,9 +25,9 @@ import type {
import { buildMatrixQaMessageContent } from "./client-message-content.js";
import {
MATRIX_QA_E2EE_SYNC_FILTER,
createMatrixQaE2eeClientLifecycle,
createMatrixQaE2eeObservedEventRecorder,
prepareMatrixQaE2eeStorage,
runMatrixQaE2eeClientOperation,
type MatrixQaE2eeActorId,
} from "./e2ee-client-internals.js";
import { findMatrixQaObservedEventMatch, normalizeMatrixQaObservedEvent } from "./events.js";
@@ -335,10 +335,22 @@ export async function createMatrixQaE2eeScenarioClient(
};
client.on("verification.summary", recordVerificationSummary);
const shutdownTimeoutMs = Math.max(1, Math.min(10_000, params.timeoutMs));
const lifecycle = createMatrixQaE2eeClientLifecycle({
detachListeners: () => {
client.off("room.message", recordEvent);
client.off("verification.summary", recordVerificationSummary);
},
drainPendingDecryptions: () => client.drainPendingDecryptions(),
shutdownTimeoutMs,
stopAndPersist: () => client.stopAndPersist(),
stopWithoutPersist: () => client.stopWithoutPersist(),
});
try {
await client.start({ readyTimeoutMs: Math.min(45_000, Math.max(15_000, params.timeoutMs)) });
} catch (error) {
await client.stopAndPersist().catch(() => undefined);
await lifecycle.stop().catch(() => undefined);
throw error;
}
@@ -387,10 +399,9 @@ export async function createMatrixQaE2eeScenarioClient(
return client.crypto;
};
const runClientOperation = <T>(label: string, run: () => Promise<T>) =>
runMatrixQaE2eeClientOperation({
lifecycle.runOperation({
label,
run,
stop: () => client.stopWithoutPersist(),
timeoutMs: params.timeoutMs,
});
@@ -493,12 +504,7 @@ export async function createMatrixQaE2eeScenarioClient(
async startVerification(id, method) {
return await requireCrypto().startVerification(id, method);
},
async stop() {
await client.drainPendingDecryptions().catch(() => undefined);
client.off("room.message", recordEvent);
client.off("verification.summary", recordVerificationSummary);
await client.stopAndPersist();
},
stop: lifecycle.stop,
waitForOptionalRoomEvent,
async waitForRoomEvent(waitParams) {
const result = await waitForOptionalRoomEvent(waitParams);