From 15ae1ddb9303995e15289ce728bf92eee65f0ece Mon Sep 17 00:00:00 2001 From: chenyangjun-xy Date: Tue, 28 Jul 2026 06:03:09 +0800 Subject: [PATCH] fix(plugin-sdk): reject malformed UTF-8 in live provider model catalog responses (#111766) * fix: reject malformed UTF-8 in live provider model catalog responses Add { fatal: true } to TextDecoder in readLiveModelCatalogJson so invalid UTF-8 bytes throw a TypeError before JSON.parse instead of silently becoming U+FFFD. Co-Authored-By: Claude Opus 4.8 (1M context) * test(plugin-sdk): add fallback proof for malformed UTF-8 in live catalog Adds a focused test that constructs raw bytes with an invalid 0xFE byte inside a JSON string value and validates that buildLiveModelProviderConfig catches the Typeerror and falls back to the static catalog rows. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(pr): classify auto-merge failures without ripgrep --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Peter Steinberger --- scripts/pr-lib/merge.sh | 27 ++++++++++-- .../provider-catalog-live-runtime.test.ts | 42 +++++++++++++++++++ .../provider-catalog-live-runtime.ts | 2 +- 3 files changed, 67 insertions(+), 4 deletions(-) diff --git a/scripts/pr-lib/merge.sh b/scripts/pr-lib/merge.sh index 0170a7cf4302..2417e1f0e9c0 100644 --- a/scripts/pr-lib/merge.sh +++ b/scripts/pr-lib/merge.sh @@ -28,9 +28,30 @@ print_file_list_with_limit() { auto_merge_unavailable_error() { local log_file="$1" - grep -Eqi -- \ - 'auto[- ]merge.*(not allowed|not enabled|not available|unavailable|not configured|not supported|must be enabled)|(not allowed|not enabled|not available|unavailable|not configured|not supported).*auto[- ]merge' \ - "$log_file" + local auto_merge_pattern='[Aa][Uu][Tt][Oo][ -][Mm][Ee][Rr][Gg][Ee]' + local unavailable_pattern + local line + + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + *$auto_merge_pattern*) + for unavailable_pattern in \ + '[Nn][Oo][Tt] [Aa][Ll][Ll][Oo][Ww][Ee][Dd]' \ + '[Nn][Oo][Tt] [Ee][Nn][Aa][Bb][Ll][Ee][Dd]' \ + '[Nn][Oo][Tt] [Aa][Vv][Aa][Ii][Ll][Aa][Bb][Ll][Ee]' \ + '[Uu][Nn][Aa][Vv][Aa][Ii][Ll][Aa][Bb][Ll][Ee]' \ + '[Nn][Oo][Tt] [Cc][Oo][Nn][Ff][Ii][Gg][Uu][Rr][Ee][Dd]' \ + '[Nn][Oo][Tt] [Ss][Uu][Pp][Pp][Oo][Rr][Tt][Ee][Dd]' \ + '[Mm][Uu][Ss][Tt] [Bb][Ee] [Ee][Nn][Aa][Bb][Ll][Ee][Dd]'; do + case "$line" in + *$unavailable_pattern*) return 0 ;; + esac + done + ;; + esac + done < "$log_file" + + return 1 } mainline_drift_requires_sync() { diff --git a/src/plugin-sdk/provider-catalog-live-runtime.test.ts b/src/plugin-sdk/provider-catalog-live-runtime.test.ts index 4a8fbe15b8ed..f7d72be6a583 100644 --- a/src/plugin-sdk/provider-catalog-live-runtime.test.ts +++ b/src/plugin-sdk/provider-catalog-live-runtime.test.ts @@ -675,6 +675,48 @@ describe("provider-catalog-live-runtime", () => { expect(release).toHaveBeenCalledTimes(1); }); + it("rejects malformed UTF-8 bytes in live catalog responses and falls back to static rows", async () => { + // Build raw bytes with a 0xFE byte inside the JSON payload — 0xFE is never + // a valid UTF-8 lead byte, so fatal:true throws before JSON.parse. + const encoder = new TextEncoder(); + const prefix = encoder.encode('{"data":[{"id":"model-a","label":"test-'); + const suffix = encoder.encode('"}]}'); + const body = new Uint8Array(prefix.length + 1 + suffix.length); + body.set(prefix, 0); + // Inject an invalid UTF-8 byte before the suffix + body[prefix.length] = 0xfe; + body.set(suffix, prefix.length + 1); + + const release = vi.fn(async () => undefined); + const fetchGuardMock: MockedFunction = vi.fn(async () => ({ + response: new Response(body), + finalUrl: "https://provider.example.test/v1/models", + release, + })); + + const providerConfig = { + api: "openai-completions" as const, + baseUrl: "https://provider.example.test/v1", + }; + const models = [buildModel("model-a"), buildModel("model-b")]; + + const result = await buildLiveModelProviderConfig({ + providerId: "provider", + endpoint: "https://provider.example.test/v1/models", + providerConfig, + apiKey: "PROVIDER_API_KEY", + fetchGuard: fetchGuardMock, + models, + }); + + // The malformed UTF-8 causes readLiveModelCatalogJson to throw. + // buildLiveModelProviderConfig should catch it and return the static catalog. + expect(result.models.map((m) => m.id)).toEqual(["model-a", "model-b"]); + expect(result.apiKey).toBe("PROVIDER_API_KEY"); + expect(fetchGuardMock).toHaveBeenCalledTimes(1); + expect(release).toHaveBeenCalledTimes(1); + }); + it("caches live provider configs and falls back to static rows on failure", async () => { const { fetchGuard, fetchGuardMock } = buildFetchGuard([ { id: "model-b", object: "model" }, diff --git a/src/plugin-sdk/provider-catalog-live-runtime.ts b/src/plugin-sdk/provider-catalog-live-runtime.ts index 354d1ac1c686..edbb4b2f29ee 100644 --- a/src/plugin-sdk/provider-catalog-live-runtime.ts +++ b/src/plugin-sdk/provider-catalog-live-runtime.ts @@ -207,7 +207,7 @@ async function readLiveModelCatalogJson(response: Response, timeoutMs: number): onIdleTimeout: ({ chunkTimeoutMs }) => new Error(`Live model catalog response stalled: no data received for ${chunkTimeoutMs}ms`), }); - return JSON.parse(new TextDecoder().decode(buffer)); + return JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(buffer)); } function readLiveModelCatalogString(value: unknown): string | undefined {