From 82ba647673f90c7193c627b5de710980789eca85 Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Tue, 11 Aug 2026 22:22:32 +0530 Subject: [PATCH] fix(media): record per-attachment media dispositions and render bounded skip markers (#122098) Inbound image, audio, and video attachments could vanish with no recorded outcome and no model-visible explanation: attachments dropped by first-only selection, disabled or unconfigured capabilities, stage failures, URL-only images, and ACP-undelivered images all left the model unaware the media existed. Fixes #122044; completes the outcome custody started for documents in #122025. Media capabilities now record one closed disposition per pre-truncation candidate at the decision site that owns the reason, with a memoized lazy native-vision probe (fires only when a marker could render; failure yields unknown and never alters outcomes; explicit image models never pay a catalog lookup). One late reader renders bounded, actionable markers from recorded facts only: native-vision and ACP-delivered images suppress per verified index, recorded failures always render, and document plus media markers share one five-marker budget with a reason-neutral overflow summary. The shipped SDK decision type stays additive (optional dispositions map). Proof: 906 + 103 focused tests including nine adversarial-review regressions, ClawSweeper local review clean (round 9, zero findings, security cleared), live Telegram image drive showing ref preservation with no false marker. Co-authored-by: Ayaan Zaidi --- .../agent-harness-runtime.json | 2 +- .../agent-harness.json | 2 +- .../agent-runtime.json | 2 +- .../channel-contract.json | 2 +- .../plugin-sdk-api-baseline/channel-core.json | 2 +- .../channel-entry-contract.json | 2 +- .../channel-inbound.json | 2 +- .../channel-message.json | 2 +- .../channel-outbound.json | 2 +- .../channel-pairing.json | 2 +- .../channel-plugin-common.json | 2 +- .../command-auth-native.json | 2 +- .../plugin-sdk-api-baseline/command-auth.json | 2 +- .../config-runtime.json | 2 +- .../conversation-runtime.json | 2 +- .../plugin-sdk-api-baseline/core.json | 2 +- .../plugin-sdk-api-baseline/discord.json | 2 +- .../plugin-sdk-api-baseline/hook-runtime.json | 2 +- .../inbound-reply-dispatch.json | 2 +- .../media-runtime.json | 2 +- .../media-understanding-runtime.json | 2 +- .../meeting-runtime.json | 2 +- .../plugin-command-runtime.json | 2 +- .../plugin-sdk-api-baseline/plugin-entry.json | 2 +- .../plugin-runtime.json | 2 +- .../provider-catalog-runtime.json | 2 +- .../reply-dispatch-runtime.json | 2 +- .../reply-runtime.json | 2 +- .../runtime-store.json | 2 +- .../session-catalog.json | 2 +- .../session-store-runtime.json | 2 +- .../plugin-sdk-api-baseline/tool-plugin.json | 2 +- .../webhook-ingress.json | 2 +- docs/nodes/media-understanding.md | 4 +- src/auto-reply/media-note.test.ts | 3 + .../media-understanding.test-fixtures.ts | 2 + src/auto-reply/reply/dispatch-acp.test.ts | 43 ++++ src/auto-reply/reply/dispatch-acp.ts | 13 +- src/auto-reply/status.test.ts | 25 +- src/media-understanding/apply-capability.ts | 25 +- src/media-understanding/apply.test.ts | 142 +++++++++-- src/media-understanding/apply.ts | 108 ++++++-- .../attachments.guards.test.ts | 23 +- src/media-understanding/attachments.select.ts | 13 +- .../file-attachment-outcomes.ts | 10 - .../media-attachment-outcomes.ts | 32 +++ .../runner.entries.guards.test.ts | 2 + src/media-understanding/runner.ts | 235 +++++++++++------- .../runner.vision-skip.test.ts | 174 +++++++++++++ src/media-understanding/runtime.test.ts | 8 +- src/media-understanding/runtime.ts | 23 +- src/media-understanding/types.ts | 15 ++ 52 files changed, 765 insertions(+), 201 deletions(-) create mode 100644 src/media-understanding/media-attachment-outcomes.ts diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json index 80baca21962c..126fa664c34d 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness-runtime.json @@ -1 +1 @@ -{"contentHash":"016fc07ce752591ba33863e49816aede24a0e90d240205dd87c882b64c611c19","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} +{"contentHash":"6b0a039cf6246df1ec52695441900203343ddc3862ef0c9c100f08cab004bfe7","entrypoint":"agent-harness-runtime","importSpecifier":"openclaw/plugin-sdk/agent-harness-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json index 5206b78aaa7d..d3d3ae4bf5b5 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-harness.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-harness.json @@ -1 +1 @@ -{"contentHash":"065fb0dd1605fb70dd75335c193f378afa8230538ce051197068e38fd24ec56d","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} +{"contentHash":"817cf688ac6ab0bcde36b59ba9d20e6511fdc82f58f727557cd33097c49b1940","entrypoint":"agent-harness","importSpecifier":"openclaw/plugin-sdk/agent-harness"} diff --git a/docs/.generated/plugin-sdk-api-baseline/agent-runtime.json b/docs/.generated/plugin-sdk-api-baseline/agent-runtime.json index 405597317ed3..3dff70f5ecb4 100644 --- a/docs/.generated/plugin-sdk-api-baseline/agent-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/agent-runtime.json @@ -1 +1 @@ -{"contentHash":"96db2cbf307a1fb22e9374a85a41da49ac538b0affe7ac40d9fcc99bcd646277","entrypoint":"agent-runtime","importSpecifier":"openclaw/plugin-sdk/agent-runtime"} +{"contentHash":"f38505150fe39946aa11b2963190d3c4838783086be6c76bbca9194447afaa63","entrypoint":"agent-runtime","importSpecifier":"openclaw/plugin-sdk/agent-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-contract.json b/docs/.generated/plugin-sdk-api-baseline/channel-contract.json index beab1506f6ca..3d78c4c4b701 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-contract.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-contract.json @@ -1 +1 @@ -{"contentHash":"75935c1d837265d191436d43bbfe24c6328b35a45a68cad250c157923badf857","entrypoint":"channel-contract","importSpecifier":"openclaw/plugin-sdk/channel-contract"} +{"contentHash":"d5872fb0a904acd1d4ed830b2597fe578daebaa855f1949bea76db923c35c373","entrypoint":"channel-contract","importSpecifier":"openclaw/plugin-sdk/channel-contract"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-core.json b/docs/.generated/plugin-sdk-api-baseline/channel-core.json index 62b59e1384d2..4ecf74aad9e7 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-core.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-core.json @@ -1 +1 @@ -{"contentHash":"59fe18c5c65144cadbfc61d22526e1475f7afb19f56303d0a007a688242e3629","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} +{"contentHash":"3d6ce8758b33237aad8417ec2731c2677cabc092f3b71671b4df28663f21c2f7","entrypoint":"channel-core","importSpecifier":"openclaw/plugin-sdk/channel-core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json index 2fdb8366afbc..af0b9b27dd08 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-entry-contract.json @@ -1 +1 @@ -{"contentHash":"96431e828c380418be9f996dcf41cdebd4c8a7c6052476dbbce519ef21dd2c9c","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} +{"contentHash":"c7f13d09cb45ae35ce5855ed7576905ca2ab4cca11b589647e8bbf84ce8d8ded","entrypoint":"channel-entry-contract","importSpecifier":"openclaw/plugin-sdk/channel-entry-contract"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json b/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json index a461dfb52537..8204b599222d 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-inbound.json @@ -1 +1 @@ -{"contentHash":"99cee44d09759fdbb01bc1b50e430933c61c745ec3c7880ea7e2aa63b382c673","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"} +{"contentHash":"d4b4c9c5f145aa60c35f928f090f74959c6c3f1745fe164bb8146b6d341e8d88","entrypoint":"channel-inbound","importSpecifier":"openclaw/plugin-sdk/channel-inbound"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-message.json b/docs/.generated/plugin-sdk-api-baseline/channel-message.json index 32fd56e42623..4021685b18c2 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-message.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-message.json @@ -1 +1 @@ -{"contentHash":"13304005d532baf3d6edccf81a594e8a874ce4ebc80174b52562a3cbf2e7a12c","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} +{"contentHash":"d2e3782705a71a08616f0c59ac1be8d8e03b1ec004c604bcc80a17567613dd71","entrypoint":"channel-message","importSpecifier":"openclaw/plugin-sdk/channel-message"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json index 7870d1970d43..7ae0b4238c81 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-outbound.json @@ -1 +1 @@ -{"contentHash":"a44717708258e2b8639874737bbc01a5ca55b0237229bf2921d7806767482156","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} +{"contentHash":"7a6122d34d21e433c7a1a0996879c0964b3a22f508a793a53ffe6404ddf4b5ef","entrypoint":"channel-outbound","importSpecifier":"openclaw/plugin-sdk/channel-outbound"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-pairing.json b/docs/.generated/plugin-sdk-api-baseline/channel-pairing.json index 3a770226c11d..1d795b91ddc3 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-pairing.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-pairing.json @@ -1 +1 @@ -{"contentHash":"fee97710da33e100a8987aed897be17ab3ec4ece9fa09b7eb871a90461c8fca3","entrypoint":"channel-pairing","importSpecifier":"openclaw/plugin-sdk/channel-pairing"} +{"contentHash":"16fa9e31deb0b1f53caedf3ec5ab54f4c1f0ab62c2e6816cec9f2abf31782b65","entrypoint":"channel-pairing","importSpecifier":"openclaw/plugin-sdk/channel-pairing"} diff --git a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json index d36eb41ff801..8d04a7ccde64 100644 --- a/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json +++ b/docs/.generated/plugin-sdk-api-baseline/channel-plugin-common.json @@ -1 +1 @@ -{"contentHash":"e974ac587d75b3a499215151dc2874d469168268a18cc024bc5048ecadf2de3b","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} +{"contentHash":"f4a7174aefe70f8bf04bfb293ca92ab9f0f99b6073a8149eee45ce902b99b5ee","entrypoint":"channel-plugin-common","importSpecifier":"openclaw/plugin-sdk/channel-plugin-common"} diff --git a/docs/.generated/plugin-sdk-api-baseline/command-auth-native.json b/docs/.generated/plugin-sdk-api-baseline/command-auth-native.json index 50c4ac785690..c85d3601c4e6 100644 --- a/docs/.generated/plugin-sdk-api-baseline/command-auth-native.json +++ b/docs/.generated/plugin-sdk-api-baseline/command-auth-native.json @@ -1 +1 @@ -{"contentHash":"257ad1c723f513e2346193c87c6f886fae528f048cf2d3ef1bb1dd41b74ea1bb","entrypoint":"command-auth-native","importSpecifier":"openclaw/plugin-sdk/command-auth-native"} +{"contentHash":"e17ce7a27cdf887381927cc2e39300105c26f72fccd34da5fead687237f022be","entrypoint":"command-auth-native","importSpecifier":"openclaw/plugin-sdk/command-auth-native"} diff --git a/docs/.generated/plugin-sdk-api-baseline/command-auth.json b/docs/.generated/plugin-sdk-api-baseline/command-auth.json index 9cc51e49d841..fe1c4c23b2e8 100644 --- a/docs/.generated/plugin-sdk-api-baseline/command-auth.json +++ b/docs/.generated/plugin-sdk-api-baseline/command-auth.json @@ -1 +1 @@ -{"contentHash":"da6ecbd54c2a221465aa717896006354000ca530d89fa772aacd60f88b7c5d6d","entrypoint":"command-auth","importSpecifier":"openclaw/plugin-sdk/command-auth"} +{"contentHash":"0e7eb50f296f755cbde5e14e06b754af5e6305202dce138a2d1835ce06ea2509","entrypoint":"command-auth","importSpecifier":"openclaw/plugin-sdk/command-auth"} diff --git a/docs/.generated/plugin-sdk-api-baseline/config-runtime.json b/docs/.generated/plugin-sdk-api-baseline/config-runtime.json index 287dc47b8edc..ddade9a935b2 100644 --- a/docs/.generated/plugin-sdk-api-baseline/config-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/config-runtime.json @@ -1 +1 @@ -{"contentHash":"40df7e972484ae28718fef1095b2b44cb2cc9eadd35464bf28d4ae0d0f0f66b5","entrypoint":"config-runtime","importSpecifier":"openclaw/plugin-sdk/config-runtime"} +{"contentHash":"0032a0767e4ddf214eb517cfe7352cd3c682a936d8c33e5602b0a9813052b11d","entrypoint":"config-runtime","importSpecifier":"openclaw/plugin-sdk/config-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/conversation-runtime.json b/docs/.generated/plugin-sdk-api-baseline/conversation-runtime.json index 493fb8c7ac9a..6be10343a0d7 100644 --- a/docs/.generated/plugin-sdk-api-baseline/conversation-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/conversation-runtime.json @@ -1 +1 @@ -{"contentHash":"7a5a2f4d3e5e18d211480286d125419ecbf8175bb6f8eeee9a6ee105df03cb3f","entrypoint":"conversation-runtime","importSpecifier":"openclaw/plugin-sdk/conversation-runtime"} +{"contentHash":"3bc769bb7d54a8d369e4922d24b1f7dd48b8a6df296457ee7c5bb241ab330e8b","entrypoint":"conversation-runtime","importSpecifier":"openclaw/plugin-sdk/conversation-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/core.json b/docs/.generated/plugin-sdk-api-baseline/core.json index 93f363e2dfc3..525c19dbdc99 100644 --- a/docs/.generated/plugin-sdk-api-baseline/core.json +++ b/docs/.generated/plugin-sdk-api-baseline/core.json @@ -1 +1 @@ -{"contentHash":"6a56259993260770aa4d78eed69f1115f218c46a12a599d429e92304df4ddd0a","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} +{"contentHash":"c5756c96150425246ed16313fdfc11e6aa780634018a57e920fa5d7d43fcd469","entrypoint":"core","importSpecifier":"openclaw/plugin-sdk/core"} diff --git a/docs/.generated/plugin-sdk-api-baseline/discord.json b/docs/.generated/plugin-sdk-api-baseline/discord.json index cfd7fbd67364..10ba4c612a46 100644 --- a/docs/.generated/plugin-sdk-api-baseline/discord.json +++ b/docs/.generated/plugin-sdk-api-baseline/discord.json @@ -1 +1 @@ -{"contentHash":"a7ac16bfa85588ced0e46de7f75bf694a059c2af2995de3cd3f34b907dffca25","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} +{"contentHash":"d01d8e0dd8da33679bf50faa64244e5c2c95f2c133ffc73f587e0b43176c43dd","entrypoint":"discord","importSpecifier":"openclaw/plugin-sdk/discord"} diff --git a/docs/.generated/plugin-sdk-api-baseline/hook-runtime.json b/docs/.generated/plugin-sdk-api-baseline/hook-runtime.json index 8f286e02cfd5..bea2ab24ced1 100644 --- a/docs/.generated/plugin-sdk-api-baseline/hook-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/hook-runtime.json @@ -1 +1 @@ -{"contentHash":"273ebd4fcac3041e3a136410fa396b9704c73eedb66414723996127ed80db8f3","entrypoint":"hook-runtime","importSpecifier":"openclaw/plugin-sdk/hook-runtime"} +{"contentHash":"909f37120c3e4834b772c5abf720aed63a2fc541ca4e9814360065533e925059","entrypoint":"hook-runtime","importSpecifier":"openclaw/plugin-sdk/hook-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json index 0f5286f7c055..1c7a55a32c70 100644 --- a/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json +++ b/docs/.generated/plugin-sdk-api-baseline/inbound-reply-dispatch.json @@ -1 +1 @@ -{"contentHash":"8bb9ea4976ff3657f3207838e2de5f3ce92d8c32c10bfbe8a3508e53edd16f3e","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} +{"contentHash":"be64b90e0781977ab81a78be2238569ea7d51555b8784a2be600cac60f8422e0","entrypoint":"inbound-reply-dispatch","importSpecifier":"openclaw/plugin-sdk/inbound-reply-dispatch"} diff --git a/docs/.generated/plugin-sdk-api-baseline/media-runtime.json b/docs/.generated/plugin-sdk-api-baseline/media-runtime.json index be5b5a15b9f1..d0de59f5b45c 100644 --- a/docs/.generated/plugin-sdk-api-baseline/media-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/media-runtime.json @@ -1 +1 @@ -{"contentHash":"2590b051b0e68c2e560aea3301e603ef3546de3862757f68a60b44ece3584933","entrypoint":"media-runtime","importSpecifier":"openclaw/plugin-sdk/media-runtime"} +{"contentHash":"0c1524e093eb80f9f2bf71e2ae6814b5e521b546d2f8bb381fe278190d0a2932","entrypoint":"media-runtime","importSpecifier":"openclaw/plugin-sdk/media-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/media-understanding-runtime.json b/docs/.generated/plugin-sdk-api-baseline/media-understanding-runtime.json index 6dbd2d26abc4..ebbd363aee30 100644 --- a/docs/.generated/plugin-sdk-api-baseline/media-understanding-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/media-understanding-runtime.json @@ -1 +1 @@ -{"contentHash":"f48e34067ab9b25a232af71d8ce17f9cbf9165ebce0832deaae3c1d9c9aeaab7","entrypoint":"media-understanding-runtime","importSpecifier":"openclaw/plugin-sdk/media-understanding-runtime"} +{"contentHash":"24ed026c51d147a2959964e13fd0ef3e43cc44cc808d581f877f818e66ac4c04","entrypoint":"media-understanding-runtime","importSpecifier":"openclaw/plugin-sdk/media-understanding-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json index 11bfad55f62d..36990d85f135 100644 --- a/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/meeting-runtime.json @@ -1 +1 @@ -{"contentHash":"e8a8cbcfc3bf20a51c93896e6add73d0c6d3677a7bac87ed9537afc7880c66f9","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} +{"contentHash":"d2fd18514981c62a3f5e2937b8fc42f9679d7cec9c133198754dfacf54bdfd33","entrypoint":"meeting-runtime","importSpecifier":"openclaw/plugin-sdk/meeting-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-command-runtime.json b/docs/.generated/plugin-sdk-api-baseline/plugin-command-runtime.json index f51145ad70ab..ef98be4c926b 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-command-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-command-runtime.json @@ -1 +1 @@ -{"contentHash":"e43fcfe8f88cf8d338c3e3496f3bdd6ec14afc1fcf341e171f7758f5c0190d5b","entrypoint":"plugin-command-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-command-runtime"} +{"contentHash":"3d8b13b477b5468df65ba2e282d73a99bf6e4422fe71a3811be8456318e79688","entrypoint":"plugin-command-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-command-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json index 0d67e7c394dc..867be85f536b 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-entry.json @@ -1 +1 @@ -{"contentHash":"d90b89f845a5df44d023e9455ff4d1fcbe51d4780a1223d624177d0ed639b0ff","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} +{"contentHash":"4474179d9e5549947f7ef57a950be9ad426193c8ce9f23187d5d7c535f3d4af5","entrypoint":"plugin-entry","importSpecifier":"openclaw/plugin-sdk/plugin-entry"} diff --git a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json index 46478da7b8c6..3bf619186d9a 100644 --- a/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/plugin-runtime.json @@ -1 +1 @@ -{"contentHash":"e7d2b6aa537db4452e8034bb57ae31bae15e471821fff539770a681d0d80b452","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} +{"contentHash":"91fce99cea0d4ed66d7971f246ab66e0ec2c139caeb78c89b21a434908e2ca9f","entrypoint":"plugin-runtime","importSpecifier":"openclaw/plugin-sdk/plugin-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json index ac740846107e..bf98f2596ab9 100644 --- a/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/provider-catalog-runtime.json @@ -1 +1 @@ -{"contentHash":"7095620f46028ce52847f310ccac8583d69aed6223eb222b961f6012fa626ea7","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} +{"contentHash":"927ae41bc0eeb5a07685238f4732c0caa639e28e10358555eba1096d22121172","entrypoint":"provider-catalog-runtime","importSpecifier":"openclaw/plugin-sdk/provider-catalog-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/reply-dispatch-runtime.json b/docs/.generated/plugin-sdk-api-baseline/reply-dispatch-runtime.json index e16e78fc80c2..2cfbc9063122 100644 --- a/docs/.generated/plugin-sdk-api-baseline/reply-dispatch-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/reply-dispatch-runtime.json @@ -1 +1 @@ -{"contentHash":"2ffe953c6f47f15c3357af51a34385942165158724ccf4d0a272a7c060f9a2d9","entrypoint":"reply-dispatch-runtime","importSpecifier":"openclaw/plugin-sdk/reply-dispatch-runtime"} +{"contentHash":"6b712448d5e6af99cee968aea960f12f9e6b385bf3bbb34d925720ebcd252d5f","entrypoint":"reply-dispatch-runtime","importSpecifier":"openclaw/plugin-sdk/reply-dispatch-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/reply-runtime.json b/docs/.generated/plugin-sdk-api-baseline/reply-runtime.json index c0da16c512d1..64095350220a 100644 --- a/docs/.generated/plugin-sdk-api-baseline/reply-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/reply-runtime.json @@ -1 +1 @@ -{"contentHash":"ca4c1916f393a407fb4f5ad3a0fedc4bc66a8f87f1f38806ffb22928f2b57a2e","entrypoint":"reply-runtime","importSpecifier":"openclaw/plugin-sdk/reply-runtime"} +{"contentHash":"cb54b3d858974c81cc1bd07e0a72b543e7f0a47f3751bd0a287c2f97bbb2b9ac","entrypoint":"reply-runtime","importSpecifier":"openclaw/plugin-sdk/reply-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/runtime-store.json b/docs/.generated/plugin-sdk-api-baseline/runtime-store.json index a900a0249777..f47f42a76b13 100644 --- a/docs/.generated/plugin-sdk-api-baseline/runtime-store.json +++ b/docs/.generated/plugin-sdk-api-baseline/runtime-store.json @@ -1 +1 @@ -{"contentHash":"64e6bc07bb893d607cc4a194f85cbeedee23ec448edd40eb6e2757a9249ec4e0","entrypoint":"runtime-store","importSpecifier":"openclaw/plugin-sdk/runtime-store"} +{"contentHash":"17e1939c2b38eba30dcd117a8448a54760b29cb13bd3e0cc9beb2795b0dee438","entrypoint":"runtime-store","importSpecifier":"openclaw/plugin-sdk/runtime-store"} diff --git a/docs/.generated/plugin-sdk-api-baseline/session-catalog.json b/docs/.generated/plugin-sdk-api-baseline/session-catalog.json index 0b1715d833bf..55123550f743 100644 --- a/docs/.generated/plugin-sdk-api-baseline/session-catalog.json +++ b/docs/.generated/plugin-sdk-api-baseline/session-catalog.json @@ -1 +1 @@ -{"contentHash":"c6fb826551fd871ce2a2227ab734aa19296a590666f4a5a52d6ce2761a06fcbe","entrypoint":"session-catalog","importSpecifier":"openclaw/plugin-sdk/session-catalog"} +{"contentHash":"838421524189909bb222d8249ede8f53dadfdf9c73a32258c5287b35a2db21d5","entrypoint":"session-catalog","importSpecifier":"openclaw/plugin-sdk/session-catalog"} diff --git a/docs/.generated/plugin-sdk-api-baseline/session-store-runtime.json b/docs/.generated/plugin-sdk-api-baseline/session-store-runtime.json index a71a18e77449..04bb46758cbb 100644 --- a/docs/.generated/plugin-sdk-api-baseline/session-store-runtime.json +++ b/docs/.generated/plugin-sdk-api-baseline/session-store-runtime.json @@ -1 +1 @@ -{"contentHash":"4626133e56c25b82d2e014f3f0e6fa2862162d603cfe57fd208f0b8d4ce89a3e","entrypoint":"session-store-runtime","importSpecifier":"openclaw/plugin-sdk/session-store-runtime"} +{"contentHash":"c9b4b25729f5edb282348b1509c2fe943ba2a9aeacccf5392678033528329ad3","entrypoint":"session-store-runtime","importSpecifier":"openclaw/plugin-sdk/session-store-runtime"} diff --git a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json index d5df39a85fc0..b334dd770a14 100644 --- a/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json +++ b/docs/.generated/plugin-sdk-api-baseline/tool-plugin.json @@ -1 +1 @@ -{"contentHash":"6738f9212ab1ca6c481cc9f3cea782baab29d63753770db25e39d83233d4f0f7","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} +{"contentHash":"7b9e970e85cb5ac7d407bd5bf70e85b38407ba9d26b66a5bb8d830623cf806d7","entrypoint":"tool-plugin","importSpecifier":"openclaw/plugin-sdk/tool-plugin"} diff --git a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json index cba3850ae82a..8b1d167e7823 100644 --- a/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json +++ b/docs/.generated/plugin-sdk-api-baseline/webhook-ingress.json @@ -1 +1 @@ -{"contentHash":"ac368b3e61d2b5d15aeaf726b9e3816aa9bc44558a4ab7e738b4094276fb1fe5","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} +{"contentHash":"5cfa7051b82c9d49810529f1522f584f28bc45b184416b8a898017423ebba415","entrypoint":"webhook-ingress","importSpecifier":"openclaw/plugin-sdk/webhook-ingress"} diff --git a/docs/nodes/media-understanding.md b/docs/nodes/media-understanding.md index 9c70b8a3622d..2c512a139ec7 100644 --- a/docs/nodes/media-understanding.md +++ b/docs/nodes/media-understanding.md @@ -265,8 +265,10 @@ When `mode: "all"`, outputs are labeled `[Image 1/2]`, `[Audio 2/2]`, etc. - Read failures get `[Attachment could not be read]`. - URL attachments get `[Attachment skipped: URL file sources are disabled]` when URL file sources are disabled. - A file with no extractable text gets `[No extractable text]`. -- At most five skip markers render per message; further skipped attachments collapse into one reason-neutral `[ more attachments skipped]` summary so junk attachments cannot grow the prompt without bound. +- At most five skip markers render per message; further skipped attachments collapse into one reason-neutral `[ more attachments skipped]` summary so junk attachments cannot grow the prompt without bound. File and image, audio, or video markers share this five-marker budget. - If a PDF falls back to rendered page images, OpenClaw forwards those images to vision-capable reply models and keeps the placeholder `[PDF content rendered to images]` in the file block. +- Image, audio, and video decisions record one closed disposition for every attachment candidate: handled, handed to native vision, not selected after the attachment limit, disabled, missing a model, denied by chat scope, or failed. +- Unhandled media gets a bounded model-visible marker. Images handed to native vision and media turns owned by another harness do not add markers. ## Config examples diff --git a/src/auto-reply/media-note.test.ts b/src/auto-reply/media-note.test.ts index fb0968c504d0..0c9ba3392332 100644 --- a/src/auto-reply/media-note.test.ts +++ b/src/auto-reply/media-note.test.ts @@ -108,6 +108,8 @@ describe("buildInboundMediaNote", () => { { capability: "image", outcome: "skipped", + attachmentDispositions: { 0: { kind: "failed" } }, + nativeVisionActive: false, attachments: [ { attachmentIndex: 0, @@ -249,6 +251,7 @@ describe("buildInboundMediaNote", () => { { capability: "audio", outcome: "success", + attachmentDispositions: { 99: { kind: "handled" } }, attachments: [ { attachmentIndex: 99, diff --git a/src/auto-reply/media-understanding.test-fixtures.ts b/src/auto-reply/media-understanding.test-fixtures.ts index d8a6b3fbcf93..2fd93486b30e 100644 --- a/src/auto-reply/media-understanding.test-fixtures.ts +++ b/src/auto-reply/media-understanding.test-fixtures.ts @@ -7,6 +7,8 @@ function createSuccessfulMediaDecision( return { capability, outcome: "success", + attachmentDispositions: { 0: { kind: "handled" } }, + ...(capability === "image" ? { nativeVisionActive: false } : {}), attachments: [ { attachmentIndex: 0, diff --git a/src/auto-reply/reply/dispatch-acp.test.ts b/src/auto-reply/reply/dispatch-acp.test.ts index c6e6d449f3ca..82a552941a07 100644 --- a/src/auto-reply/reply/dispatch-acp.test.ts +++ b/src/auto-reply/reply/dispatch-acp.test.ts @@ -1212,6 +1212,49 @@ describe("tryDispatchAcpReplyCore", () => { } }); + it("passes exactly the resolved attachment indexes as delivered images", async () => { + setReadyAcpResolution(); + mockVisibleTextTurn("image turn"); + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "dispatch-acp-")); + const imagePath = path.join(tempDir, "delivered.png"); + try { + // Real PNG bytes: the turn-attachment resolver byte-sniffs image MIME + // through the harness buffer map keyed by local path. + await fs.writeFile(imagePath, ACP_PNG_IMAGE_BYTES); + acpAttachmentBuffers.set(imagePath, ACP_PNG_IMAGE_BYTES); + + await runDispatch({ + bodyForAgent: "describe both images", + cfg: createAcpTestConfig({ + channels: { + imessage: { + attachmentRoots: [tempDir], + }, + }, + }), + ctxOverrides: { + Provider: "imessage", + Surface: "imessage", + media: [ + { path: imagePath, contentType: "image/png", kind: "image" }, + { url: "https://cdn.example.test/photos/remote.png", contentType: "image/png" }, + ], + }, + }); + + // The delivered set must mirror the resolver: local image in, remote-url + // image out — an empty or over-broad set reintroduces false skip claims. + const delivered = requireRecord( + mockArg(mediaUnderstandingMocks.applyMediaUnderstanding, 0, 0, "media understanding"), + "media understanding", + ).deliveredImageIndexes as ReadonlySet; + expect(delivered.has(0)).toBe(true); + expect(delivered.has(1)).toBe(false); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + it("selects bounded recent local history images", () => { const now = 1_700_000_000_000; const ctx = buildTestCtx({ diff --git a/src/auto-reply/reply/dispatch-acp.ts b/src/auto-reply/reply/dispatch-acp.ts index 94796b0cef47..e08c2249a560 100644 --- a/src/auto-reply/reply/dispatch-acp.ts +++ b/src/auto-reply/reply/dispatch-acp.ts @@ -707,6 +707,13 @@ export async function tryDispatchAcpReplyCore(params: { auditTerminalOutcome = "blocked"; throw agentPolicyError; } + // Resolve turn attachments before media understanding so marker rendering + // suppresses exactly the image indexes ACP will deliver with the turn. + const resolvedTurnAttachments = await resolveAgentTurnAttachments({ + ctx: params.ctx, + cfg: params.cfg, + includeAttachmentIndexes: true, + }); let extractedFileImages = params.extractedFileImages ?? []; if (hasInboundMediaForUnderstanding(params.ctx) && !params.ctx.MediaUnderstanding?.length) { try { @@ -714,6 +721,7 @@ export async function tryDispatchAcpReplyCore(params: { const mediaResult = await applyMediaUnderstanding({ ctx: params.ctx, cfg: params.cfg, + deliveredImageIndexes: new Set(resolvedTurnAttachments.attachmentIndexes ?? []), agentId: acpAgentId, agentDir: resolveAgentDir(params.cfg, acpAgentId), workspaceDir: resolveAgentWorkspaceDir(params.cfg, acpAgentId), @@ -729,11 +737,6 @@ export async function tryDispatchAcpReplyCore(params: { } const promptText = resolveAcpPromptText(params.ctx); - const resolvedTurnAttachments = await resolveAgentTurnAttachments({ - ctx: params.ctx, - cfg: params.cfg, - includeAttachmentIndexes: true, - }); const mediaAttachments = resolvedTurnAttachments.attachments; const inlineAttachments = resolveInlineAgentImageAttachments(params.images); const extractedAttachments = resolveInlineAgentImageAttachments( diff --git a/src/auto-reply/status.test.ts b/src/auto-reply/status.test.ts index fbc5786e8e0f..d42ff89c8c6d 100644 --- a/src/auto-reply/status.test.ts +++ b/src/auto-reply/status.test.ts @@ -1330,6 +1330,7 @@ describe("buildStatusMessage", () => { { capability: "audio", outcome: "skipped", + attachmentDispositions: { 1: { kind: "failed" } }, attachments: [ { attachmentIndex: 1, @@ -1360,6 +1361,7 @@ describe("buildStatusMessage", () => { { capability: "audio", outcome: "success", + attachmentDispositions: { 0: { kind: "handled" } }, attachments: [ { attachmentIndex: 0, @@ -1391,6 +1393,7 @@ describe("buildStatusMessage", () => { { capability: "audio", outcome: "failed", + attachmentDispositions: { 0: { kind: "failed" } }, attachments: [ { attachmentIndex: 0, @@ -1425,9 +1428,25 @@ describe("buildStatusMessage", () => { sessionKey: "agent:main:main", queue: { mode: "none" }, mediaDecisions: [ - { capability: "image", outcome: "no-attachment", attachments: [] }, - { capability: "audio", outcome: "no-attachment", attachments: [] }, - { capability: "video", outcome: "no-attachment", attachments: [] }, + { + capability: "image", + outcome: "no-attachment", + attachments: [], + attachmentDispositions: {}, + nativeVisionActive: false, + }, + { + capability: "audio", + outcome: "no-attachment", + attachments: [], + attachmentDispositions: {}, + }, + { + capability: "video", + outcome: "no-attachment", + attachments: [], + attachmentDispositions: {}, + }, ], }); diff --git a/src/media-understanding/apply-capability.ts b/src/media-understanding/apply-capability.ts index d1d6f5570852..605e8cd44422 100644 --- a/src/media-understanding/apply-capability.ts +++ b/src/media-understanding/apply-capability.ts @@ -1,16 +1,37 @@ // Keeps one provider failure from blocking the remaining media capabilities. import { logVerbose, shouldLogVerbose } from "../globals.js"; +import { selectAttachments } from "./attachments.js"; import { runCapability } from "./runner.js"; export async function runMediaCapability( params: Parameters[0], -): Promise> | undefined> { +): Promise>> { try { return await runCapability(params); } catch (err) { if (shouldLogVerbose()) { logVerbose(`Media understanding task failed: ${String(err)}`); } - return undefined; + const selection = selectAttachments({ + capability: params.capability, + attachments: params.media, + policy: params.config?.attachments, + }); + return { + outputs: [], + decision: { + capability: params.capability, + outcome: "failed", + attachments: [], + // Dropped attachments were never attempted; only selected ones failed. + attachmentDispositions: Object.fromEntries([ + ...selection.selected.map(({ index }) => [index, { kind: "failed" as const }] as const), + ...selection.droppedAttachmentIndexes.map( + (index) => [index, { kind: "not-selected" as const }] as const, + ), + ]), + ...(params.capability === "image" ? { nativeVisionActive: false } : {}), + }, + }; } } diff --git a/src/media-understanding/apply.test.ts b/src/media-understanding/apply.test.ts index 593a931c53b9..c906d6785b57 100644 --- a/src/media-understanding/apply.test.ts +++ b/src/media-understanding/apply.test.ts @@ -282,16 +282,6 @@ async function applyWithDisabledMedia(params: { return { ctx, result }; } -function expectFileNotApplied(params: { - ctx: MsgContext; - result: { appliedFile: boolean }; - body: string; -}) { - expect(params.result.appliedFile).toBe(false); - expect(params.ctx.Body).toBe(params.body); - expect(params.ctx.Body).not.toContain(" { expect(result.appliedAudio).toBe(false); expect(transcribeAudio).not.toHaveBeenCalled(); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe("[Audio attachment could not be analyzed]"); }); it("falls back to CLI model when provider fails", async () => { @@ -874,7 +864,7 @@ describe("applyMediaUnderstanding", () => { ); expect(ctx.Transcript).toBeUndefined(); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe("[Audio attachment could not be analyzed]"); const [command] = getRunExecCall(); expect(command).toBe("sherpa-onnx-offline"); }); @@ -1016,7 +1006,9 @@ describe("applyMediaUnderstanding", () => { ); expect(ctx.Transcript).toBeUndefined(); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe( + "[Audio attachment not analyzed: no audio-understanding model is configured]", + ); expect(mockedRunExec).not.toHaveBeenCalled(); }); @@ -1048,7 +1040,9 @@ describe("applyMediaUnderstanding", () => { ); expect(ctx.Transcript).toBeUndefined(); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe( + "[Audio attachment not analyzed: no audio-understanding model is configured]", + ); expect(mockedRunExec).not.toHaveBeenCalled(); }); @@ -1075,10 +1069,51 @@ describe("applyMediaUnderstanding", () => { expect(result.appliedImage).toBe(false); }); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe( + "[Image attachment not analyzed: no image-understanding model is configured]", + ); expect(mockedRunExec).not.toHaveBeenCalled(); }); + it("suppresses markers only for images the ACP caller actually delivers", async () => { + clearMediaUnderstandingBinaryCacheForTests(); + const binDir = await createTempMediaDir(); + await createMockExecutable(binDir, "agy"); + const deliveredPath = await createTempMediaFile({ + fileName: "delivered.jpg", + content: "image-bytes", + }); + const undeliveredPath = await createTempMediaFile({ + fileName: "undelivered.jpg", + content: "image-bytes", + }); + const ctx: MsgContext = { + Body: "", + media: [ + { path: deliveredPath, contentType: "image/jpeg" }, + { path: undeliveredPath, contentType: "image/jpeg" }, + ], + }; + const cfg: OpenClawConfig = { + tools: { media: { image: { attachments: { mode: "all", maxAttachments: 4 } } } }, + }; + mockedResolveApiKey.mockResolvedValue({ source: "none", mode: "api-key" }); + + await withMediaAutoDetectEnv({ PATH: binDir }, async () => { + const result = await applyMediaUnderstanding({ + ctx, + cfg, + deliveredImageIndexes: new Set([0]), + }); + expect(result.appliedImage).toBe(false); + }); + + // Index 0 rides with the ACP turn (no marker); index 1 was not resolved + // into an attachment, so its non-delivery stays model-visible. + const markerCount = ctx.Body?.split("[Image attachment not analyzed").length ?? 0; + expect(markerCount - 1).toBe(1); + }); + it("uses CLI image understanding and preserves caption for commands", async () => { const imagePath = await createTempMediaFile({ fileName: "photo.jpg", @@ -1270,6 +1305,56 @@ describe("applyMediaUnderstanding", () => { expect(ctx.Body).toBe("[Image]\nDescription:\nnormalized image"); }); + it("renders recorded outcomes for every image candidate when no model is configured", async () => { + const ctx: MsgContext = { + Body: "", + media: Array.from({ length: 4 }, (_, index) => ({ + path: `/tmp/photo-${index}.jpg`, + contentType: "image/jpeg", + })), + }; + + const result = await applyMediaUnderstanding({ + ctx, + cfg: { tools: { media: { image: { enabled: true } } } }, + }); + + const imageDecision = result.decisions.find((decision) => decision.capability === "image"); + expect(imageDecision).toMatchObject({ + attachmentDispositions: { + 0: { kind: "no-model" }, + 1: { kind: "not-selected" }, + 2: { kind: "not-selected" }, + 3: { kind: "not-selected" }, + }, + }); + expect(ctx.Body).toBe( + [ + "[Image attachment not analyzed: no image-understanding model is configured]", + "[Image attachment not processed: attachment limit reached]", + "[Image attachment not processed: attachment limit reached]", + "[Image attachment not processed: attachment limit reached]", + ].join("\n\n"), + ); + }); + + it("caps markers for disabled image understanding", async () => { + const ctx: MsgContext = { + Body: "", + media: Array.from({ length: 7 }, (_, index) => ({ + path: `/tmp/disabled-photo-${index}.jpg`, + contentType: "image/jpeg", + })), + }; + + await applyMediaUnderstanding({ ctx, cfg: createMediaDisabledConfig() }); + + expect( + ctx.Body?.split("[Image attachment not analyzed: image understanding is disabled]"), + ).toHaveLength(6); + expect(ctx.Body).toContain("[2 more attachments skipped]"); + }); + it("uses active model when enabled and models are missing", async () => { const audioPath = await createTempMediaFile({ fileName: "fallback.ogg", @@ -1394,6 +1479,7 @@ describe("applyMediaUnderstanding", () => { capability: "audio", outcome: "no-attachment", attachments: [], + attachmentDispositions: {}, }); }); @@ -1751,7 +1837,10 @@ describe("applyMediaUnderstanding", () => { mediaType: "audio/mpeg", }); - expectFileNotApplied({ ctx, result, body: "" }); + expect(result.appliedFile).toBe(false); + expect(ctx.Body).toBe( + "\n\n[Audio attachment not analyzed: audio understanding is disabled]", + ); }); it("reports archive container attachments with +zip MIME types as unsupported", async () => { @@ -2176,31 +2265,32 @@ describe("applyMediaUnderstanding", () => { expect(ctx.Body).toContain("[2 more attachments skipped]"); }); - it("keeps the overflow summary reason-neutral when skipped kinds are mixed", async () => { + it("shares one reason-neutral overflow budget across document and media markers", async () => { const olePayload = Buffer.from("Root Entry WordDocument legacy preview", "utf8"); const media: { path: string; contentType: string }[] = []; - for (let i = 0; i < 5; i += 1) { + for (let i = 0; i < 4; i += 1) { const filePath = await createTempMediaFile({ fileName: `mixed-legacy-${i}.doc`, content: olePayload, }); media.push({ path: filePath, contentType: "application/msword" }); } - const pdfPath = await createTempMediaFile({ - fileName: "report.pdf", - content: Buffer.from("%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n", "utf8"), - }); - media.push({ path: pdfPath, contentType: "application/pdf" }); + for (let i = 0; i < 3; i += 1) { + media.push({ path: `/tmp/junk-image-${i}.jpg`, contentType: "image/jpeg" }); + } const ctx: MsgContext = { Body: "", media }; const result = await applyMediaUnderstanding({ ctx, - cfg: createMediaDisabledConfigWithAllowedMimes(["text/plain"]), + cfg: createMediaDisabledConfig(), }); expect(result.appliedFile).toBe(true); - expect(ctx.Body).toContain("[1 more attachment skipped]"); - expect(ctx.Body).not.toContain("[Attachment type not allowed"); + expect(ctx.Body?.split("[Unsupported document format")).toHaveLength(5); + expect( + ctx.Body?.split("[Image attachment not analyzed: image understanding is disabled]"), + ).toHaveLength(2); + expect(ctx.Body).toContain("[2 more attachments skipped]"); }); it("keeps vendor +json attachments eligible for text extraction", async () => { diff --git a/src/media-understanding/apply.ts b/src/media-understanding/apply.ts index add7eb98d405..27f0d6cd1916 100644 --- a/src/media-understanding/apply.ts +++ b/src/media-understanding/apply.ts @@ -32,15 +32,18 @@ import type { ExtractedFileImage } from "./extracted-file-images.js"; import { type FileAttachmentOutcome, isSkippedFileOutcome, - MAX_SKIPPED_FILE_MARKERS, renderFileAttachmentOutcome, - renderSkippedFileOverflowSummary, sanitizeMimeType, } from "./file-attachment-outcomes.js"; import { type FileExtractionLimits, resolveFileExtractionLimits, } from "./file-extraction-limits.js"; +import { + MAX_SKIPPED_FILE_MARKERS, + renderMediaAttachmentDisposition, + renderSkippedFileOverflowSummary, +} from "./media-attachment-outcomes.js"; import { resolveConcurrency } from "./resolve.js"; import { buildProviderRegistry, @@ -184,6 +187,8 @@ type ClassifiedFileAttachment = { mimeType?: string; }; +type AttachmentContextBlock = { text: string; consumesMarkerBudget: boolean }; + // URL attachments may carry signed query credentials; only the pathname // basename is safe to surface as a model-visible display name. function attachmentUrlDisplayName(url: string): string | undefined { @@ -350,10 +355,8 @@ async function extractFileContext(params: { if (!attachments || attachments.length === 0) { return { blocks: [], images: [] }; } - const blocks: string[] = []; + const blocks: AttachmentContextBlock[] = []; const images: ExtractedFileImage[] = []; - let skippedMarkers = 0; - let skippedOverflow = 0; for (const attachment of attachments) { if (!attachment) { continue; @@ -377,28 +380,74 @@ async function extractFileContext(params: { if (blockText === null) { continue; } - if (isSkippedFileOutcome(outcome)) { - if (skippedMarkers >= MAX_SKIPPED_FILE_MARKERS) { - skippedOverflow += 1; - continue; - } - skippedMarkers += 1; - } - blocks.push( - renderFileContextBlock({ + blocks.push({ + text: renderFileContextBlock({ filename, fallbackName: `file-${attachment.index + 1}`, mimeType, content: blockText, }), - ); - } - if (skippedOverflow > 0) { - blocks.push(renderSkippedFileOverflowSummary(skippedOverflow)); + consumesMarkerBudget: isSkippedFileOutcome(outcome), + }); } return { blocks, images }; } +function renderMediaAttachmentMarkers(params: { + attachments: MediaAttachment[]; + decisions: MediaUnderstandingDecision[]; + outputs: MediaUnderstandingOutput[]; + deliveredImageIndexes?: ReadonlySet; +}): AttachmentContextBlock[] { + const handledIndexes = new Set(params.outputs.map((output) => output.attachmentIndex)); + const decisions = new Map(params.decisions.map((decision) => [decision.capability, decision])); + return params.attachments.flatMap((attachment) => { + const capability = resolveAttachmentKind(attachment); + if (capability !== "image" && capability !== "audio" && capability !== "video") { + return []; + } + // The ACP caller resolved these exact indexes into native turn attachments; + // a marker would falsely claim non-delivery. Unresolved images keep theirs. + if (capability === "image" && params.deliveredImageIndexes?.has(attachment.index)) { + return []; + } + const decision = decisions.get(capability); + if (!decision || handledIndexes.has(attachment.index)) { + return []; + } + const disposition = decision.attachmentDispositions?.[attachment.index]; + // Vision-capable model → the reply runtime hydrates images natively; an + // absence-of-processing marker would contradict what the model sees. + // Recorded per-attachment failures stay visible — they are authoritative + // regardless of native delivery. Partial/failed native hydration remains + // unexplainable at this frozen-prompt stage (#122101). + if ( + capability === "image" && + decision.nativeVisionActive !== false && + disposition?.kind !== "failed" + ) { + return []; + } + const text = disposition ? renderMediaAttachmentDisposition(capability, disposition) : null; + return text ? [{ text, consumesMarkerBudget: true }] : []; + }); +} + +function applyAttachmentMarkerBudget(blocks: AttachmentContextBlock[]): string[] { + const rendered: string[] = []; + let markers = 0; + let overflow = 0; + for (const block of blocks) { + if (block.consumesMarkerBudget && markers >= MAX_SKIPPED_FILE_MARKERS) { + overflow += 1; + continue; + } + markers += Number(block.consumesMarkerBudget); + rendered.push(block.text); + } + return overflow > 0 ? [...rendered, renderSkippedFileOverflowSummary(overflow)] : rendered; +} + export async function applyMediaUnderstanding(params: { ctx: MsgContext; cfg: OpenClawConfig; @@ -409,6 +458,8 @@ export async function applyMediaUnderstanding(params: { activeModel?: ActiveMediaModel; /** Preserve native-harness ownership of image, video, and file inputs while applying STT. */ processingMode?: "audio-only"; + /** Attachment indexes the caller (ACP) has already resolved into native turn attachments. */ + deliveredImageIndexes?: ReadonlySet; }): Promise { const { ctx, cfg } = params; const commandCandidates = [ctx.CommandBody, ctx.RawBody, ctx.Body]; @@ -451,9 +502,6 @@ export async function applyMediaUnderstanding(params: { const outputs: MediaUnderstandingOutput[] = []; const decisions: MediaUnderstandingDecision[] = []; for (const entry of results) { - if (!entry) { - continue; - } for (const output of entry.outputs) { outputs.push(output); } @@ -567,13 +615,23 @@ export async function applyMediaUnderstanding(params: { skipAttachmentIndexes: audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined, }); - if (fileContext.blocks.length > 0) { - ctx.Body = appendFileBlocks(ctx.Body, fileContext.blocks); + const mediaMarkers = + params.processingMode === "audio-only" + ? [] + : renderMediaAttachmentMarkers({ + attachments, + decisions, + outputs, + deliveredImageIndexes: params.deliveredImageIndexes, + }); + const contextBlocks = applyAttachmentMarkerBudget([...fileContext.blocks, ...mediaMarkers]); + if (contextBlocks.length > 0) { + ctx.Body = appendFileBlocks(ctx.Body, contextBlocks); } - if (outputs.length > 0 || fileContext.blocks.length > 0) { + if (outputs.length > 0 || contextBlocks.length > 0) { finalizeInboundContext(ctx, { forceBodyForAgent: true, - forceBodyForCommands: outputs.length > 0 || fileContext.blocks.length > 0, + forceBodyForCommands: true, }); } diff --git a/src/media-understanding/attachments.guards.test.ts b/src/media-understanding/attachments.guards.test.ts index 61986ed85dd4..2dde1b45c37f 100644 --- a/src/media-understanding/attachments.guards.test.ts +++ b/src/media-understanding/attachments.guards.test.ts @@ -12,7 +12,7 @@ describe("media-understanding selectAttachments guards", () => { attachments: undefined as unknown as MediaAttachment[], policy: { prefer: "path" }, }), - ).toStrictEqual([]); + ).toStrictEqual({ selected: [], droppedAttachmentIndexes: [] }); }); it("returns no selections when attachments is not an array", () => { @@ -22,7 +22,7 @@ describe("media-understanding selectAttachments guards", () => { attachments: { malformed: true } as unknown as MediaAttachment[], policy: { prefer: "url" }, }), - ).toStrictEqual([]); + ).toStrictEqual({ selected: [], droppedAttachmentIndexes: [] }); }); it("returns no selections for malformed attachment entries", () => { @@ -37,6 +37,23 @@ describe("media-understanding selectAttachments guards", () => { ] as unknown as MediaAttachment[], policy: { prefer: "path" }, }), - ).toStrictEqual([]); + ).toStrictEqual({ selected: [], droppedAttachmentIndexes: [] }); + }); + + it("reports only same-capability attachments dropped by truncation", () => { + expect( + selectAttachments({ + capability: "image", + attachments: [ + { index: 0, path: "/tmp/first.jpg", mime: "image/jpeg" }, + { index: 1, path: "/tmp/note.ogg", mime: "audio/ogg" }, + { index: 2, path: "/tmp/second.jpg", mime: "image/jpeg" }, + { index: 3, path: "/tmp/third.jpg", mime: "image/jpeg" }, + ], + }), + ).toStrictEqual({ + selected: [{ index: 0, path: "/tmp/first.jpg", mime: "image/jpeg" }], + droppedAttachmentIndexes: [2, 3], + }); }); }); diff --git a/src/media-understanding/attachments.select.ts b/src/media-understanding/attachments.select.ts index f903eb6dc727..af516c2a21e4 100644 --- a/src/media-understanding/attachments.select.ts +++ b/src/media-understanding/attachments.select.ts @@ -64,7 +64,7 @@ export function selectAttachments(params: { capability: MediaUnderstandingCapability; attachments: MediaAttachment[]; policy?: MediaUnderstandingAttachmentsConfig; -}): MediaAttachment[] { +}): { selected: MediaAttachment[]; droppedAttachmentIndexes: number[] } { const { capability, attachments, policy } = params; const input = Array.isArray(attachments) ? attachments.filter(isAttachmentRecord) : []; const matches = input.filter((item) => { @@ -81,14 +81,15 @@ export function selectAttachments(params: { return isVideoAttachment(item); }); if (matches.length === 0) { - return []; + return { selected: [], droppedAttachmentIndexes: [] }; } const ordered = orderAttachments(matches, policy?.prefer); const mode = policy?.mode ?? "first"; const maxAttachments = policy?.maxAttachments ?? DEFAULT_MAX_ATTACHMENTS; - if (mode === "all") { - return ordered.slice(0, Math.max(1, maxAttachments)); - } - return ordered.slice(0, 1); + const limit = mode === "all" ? Math.max(1, maxAttachments) : 1; + return { + selected: ordered.slice(0, limit), + droppedAttachmentIndexes: ordered.slice(limit).map((attachment) => attachment.index), + }; } diff --git a/src/media-understanding/file-attachment-outcomes.ts b/src/media-understanding/file-attachment-outcomes.ts index 211cd985a4ab..9d77fbc7c87e 100644 --- a/src/media-understanding/file-attachment-outcomes.ts +++ b/src/media-understanding/file-attachment-outcomes.ts @@ -53,10 +53,6 @@ function wrapUntrustedAttachmentContent(content: string): string { return wrapExternalContent(content, { source: "unknown", includeWarning: false }); } -// Cap cumulative skip markers so a burst of rejected attachments cannot grow -// model-visible context without bound; overflow collapses into one summary line. -export const MAX_SKIPPED_FILE_MARKERS = 5; - const SKIPPED_FILE_OUTCOME_KINDS = new Set([ "unsupported-format", "policy-rejected", @@ -68,12 +64,6 @@ export function isSkippedFileOutcome(outcome: FileAttachmentOutcome): boolean { return SKIPPED_FILE_OUTCOME_KINDS.has(outcome.kind); } -// Reason-neutral on purpose: overflow can mix unsupported, policy-rejected, -// unreadable, and url-disabled kinds; naming one reason would misdirect recovery. -export function renderSkippedFileOverflowSummary(count: number): string { - return `[${count} more attachment${count === 1 ? "" : "s"} skipped]`; -} - export function renderFileAttachmentOutcome(outcome: FileAttachmentOutcome): string | null { switch (outcome.kind) { case "extracted": diff --git a/src/media-understanding/media-attachment-outcomes.ts b/src/media-understanding/media-attachment-outcomes.ts new file mode 100644 index 000000000000..c81a18a2af04 --- /dev/null +++ b/src/media-understanding/media-attachment-outcomes.ts @@ -0,0 +1,32 @@ +import type { MediaAttachmentDisposition, MediaUnderstandingCapability } from "./types.js"; + +export const MAX_SKIPPED_FILE_MARKERS = 5; + +// Reason-neutral because the shared overflow can mix file and media outcomes. +export function renderSkippedFileOverflowSummary(count: number): string { + return `[${count} more attachment${count === 1 ? "" : "s"} skipped]`; +} + +export function renderMediaAttachmentDisposition( + capability: MediaUnderstandingCapability, + disposition: MediaAttachmentDisposition, +): string | null { + const label = `${capability[0]?.toUpperCase()}${capability.slice(1)}`; + switch (disposition.kind) { + case "handled": + case "handed-to-native-vision": + return null; + case "not-selected": + return `[${label} attachment not processed: attachment limit reached]`; + case "capability-disabled": + return `[${label} attachment not analyzed: ${capability} understanding is disabled]`; + case "no-model": + return `[${label} attachment not analyzed: no ${capability}-understanding model is configured]`; + case "scope-denied": + return `[${label} attachment not analyzed in this chat]`; + case "failed": + return `[${label} attachment could not be analyzed]`; + default: + return disposition satisfies never; + } +} diff --git a/src/media-understanding/runner.entries.guards.test.ts b/src/media-understanding/runner.entries.guards.test.ts index f3e589fa6020..74f68c3fa437 100644 --- a/src/media-understanding/runner.entries.guards.test.ts +++ b/src/media-understanding/runner.entries.guards.test.ts @@ -20,6 +20,8 @@ describe("media-understanding formatDecisionSummary guards", () => { capability: "image", outcome: "skipped", attachments: undefined as unknown as MediaUnderstandingDecision["attachments"], + attachmentDispositions: {}, + nativeVisionActive: false, }), ).toBe("image: skipped"); }); diff --git a/src/media-understanding/runner.ts b/src/media-understanding/runner.ts index 6e8febec4d45..8cedcae91074 100644 --- a/src/media-understanding/runner.ts +++ b/src/media-understanding/runner.ts @@ -34,6 +34,7 @@ import { logWarn } from "../logger.js"; import { resolveChannelInboundAttachmentRoots } from "../media/channel-inbound-roots.js"; import { getDefaultMediaLocalRoots } from "../media/local-roots.js"; import { normalizeMediaFacts } from "../media/media-facts.js"; +import { classifyMediaReferenceSource } from "../media/media-reference.js"; import { createLazyRuntimeModule, createLazyRuntimeNamedExport } from "../shared/lazy-runtime.js"; import { MediaAttachmentCache, selectAttachments } from "./attachments.js"; import { matchesMediaEntryCapability } from "./entry-capabilities.js"; @@ -60,6 +61,7 @@ import { } from "./runner.entries.js"; import type { MediaAttachment, + MediaAttachmentDisposition, MediaUnderstandingCapability, MediaUnderstandingDecision, MediaUnderstandingModelDecision, @@ -565,23 +567,15 @@ async function resolveAutoEntries(params: { providerRegistry: ProviderRegistry; capability: MediaUnderstandingCapability; activeModel?: ActiveMediaModel; + nativeVisionActive: boolean; }): Promise { - if (params.capability === "image") { - const activeSupportsVision = await activeModelSupportsNativeVision({ + if (params.capability === "image" && !params.nativeVisionActive) { + const imageModelEntries = resolveImageModelFromAgentDefaults({ cfg: params.cfg, agentId: params.agentId, - activeModel: params.activeModel, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, }); - if (!activeSupportsVision) { - const imageModelEntries = resolveImageModelFromAgentDefaults({ - cfg: params.cfg, - agentId: params.agentId, - }); - if (imageModelEntries.length > 0) { - return imageModelEntries; - } + if (imageModelEntries.length > 0) { + return imageModelEntries; } } const activeEntry = await resolveActiveModelEntry(params); @@ -840,6 +834,13 @@ function hasFailedMediaAttempt(attachments: MediaUnderstandingDecision["attachme ); } +function createAttachmentDispositions( + indexes: readonly number[], + disposition: MediaAttachmentDisposition, +): Record { + return Object.fromEntries(indexes.map((index) => [index, disposition])); +} + export async function runCapability(params: { capability: MediaUnderstandingCapability; cfg: OpenClawConfig; @@ -855,23 +856,81 @@ export async function runCapability(params: { }): Promise { const { capability, cfg, ctx } = params; const config: MediaUnderstandingConfig = params.config ?? cfg.tools?.media?.[capability] ?? {}; + const selection = selectAttachments({ + capability, + attachments: params.media, + policy: config.attachments, + }); + const selectedAttachmentIndexes = selection.selected.map((attachment) => attachment.index); + const activeProvider = params.activeModel?.provider?.trim(); + // One memoized owner for the native-vision fact. Probed lazily — only when + // the skip branch must decide, or an image decision carries a renderable + // disposition — so explicit image models never pay a catalog lookup. A probe + // failure yields "unknown" and never alters a decision outcome; unknown + // suppresses image markers because a false skip claim beside a natively + // delivered image is worse than silence (#122101). + let nativeVisionProbe: Promise | undefined; + const resolveNativeVisionFlag = (): Promise => { + nativeVisionProbe ??= activeModelSupportsNativeVision({ + cfg, + agentId: params.agentId, + activeModel: params.activeModel, + agentDir: params.agentDir, + workspaceDir: params.workspaceDir, + }).catch((err: unknown) => { + if (shouldLogVerbose()) { + logVerbose(`native vision support probe failed: ${String(err)}`); + } + return undefined; + }); + return nativeVisionProbe; + }; + const buildDispositions = ( + selectedDisposition: MediaAttachmentDisposition, + droppedDisposition = selectedDisposition, + ) => ({ + ...createAttachmentDispositions(selectedAttachmentIndexes, selectedDisposition), + ...createAttachmentDispositions(selection.droppedAttachmentIndexes, droppedDisposition), + }); + const rendersMarker = (dispositions: Record) => + Object.values(dispositions).some( + (d) => d.kind !== "handled" && d.kind !== "handed-to-native-vision", + ); + const buildDecision = async ( + outcome: MediaUnderstandingDecision["outcome"], + attachments: MediaUnderstandingDecision["attachments"], + attachmentDispositions: Record, + ): Promise => { + // Record the fact whenever it is known (probe already ran) or needed + // (a marker could render); never fire the probe for marker-free decisions. + const nativeVisionActive = + capability === "image" && + (nativeVisionProbe !== undefined || rendersMarker(attachmentDispositions)) + ? await resolveNativeVisionFlag() + : undefined; + return { + capability, + outcome, + attachments, + attachmentDispositions, + ...(nativeVisionActive !== undefined ? { nativeVisionActive } : {}), + }; + }; if (config?.enabled === false) { return { outputs: [], - decision: { capability, outcome: "disabled", attachments: [] }, + decision: await buildDecision( + "disabled", + [], + buildDispositions({ kind: "capability-disabled" }), + ), }; } - const attachmentPolicy = config?.attachments; - const selected = selectAttachments({ - capability, - attachments: params.media, - policy: attachmentPolicy, - }); - if (selected.length === 0) { + if (selection.selected.length === 0) { return { outputs: [], - decision: { capability, outcome: "no-attachment", attachments: [] }, + decision: await buildDecision("no-attachment", [], {}), }; } @@ -882,61 +941,66 @@ export async function runCapability(params: { } return { outputs: [], - decision: { - capability, - outcome: "scope-deny", - attachments: selected.map((item) => ({ attachmentIndex: item.index, attempts: [] })), - }, + decision: await buildDecision( + "scope-deny", + selection.selected.map((item) => ({ + attachmentIndex: item.index, + attempts: [], + })), + buildDispositions({ kind: "scope-denied" }), + ), }; } // Skip image understanding when the primary model supports vision natively. // The image will be injected directly into the model context instead. - const activeProvider = params.activeModel?.provider?.trim(); if ( capability === "image" && activeProvider && - !hasExplicitImageUnderstandingConfig({ - cfg, - providerRegistry: params.providerRegistry, - }) + !hasExplicitImageUnderstandingConfig({ cfg, providerRegistry: params.providerRegistry }) && + (await resolveNativeVisionFlag()) === true ) { - if ( - await activeModelSupportsNativeVision({ - cfg, - agentId: params.agentId, - activeModel: params.activeModel, - agentDir: params.agentDir, - workspaceDir: params.workspaceDir, - }) - ) { - if (shouldLogVerbose()) { - logVerbose("Skipping image understanding: primary model supports vision natively"); - } - const model = params.activeModel?.model?.trim(); - const reason = "primary model supports vision natively"; - return { - outputs: [], - decision: { - capability, - outcome: "skipped", - attachments: selected.map((item) => { - const attempt = { - type: "provider" as const, - provider: activeProvider, - model: model || undefined, - outcome: "skipped" as const, - reason, - }; - return { - attachmentIndex: item.index, - attempts: [attempt], - chosen: attempt, - }; - }), - }, - }; + if (shouldLogVerbose()) { + logVerbose("Skipping image understanding: primary model supports vision natively"); } + const model = params.activeModel?.model?.trim(); + const reason = "primary model supports vision natively"; + // Native hydration resolves local paths and media-store refs only; a + // remote-URL-only image is never delivered that way, so claiming the + // handoff would suppress its marker while it silently vanishes. + const nativeDeliverable = (item: MediaAttachment) => + Boolean(item.path) || + (Boolean(item.url) && classifyMediaReferenceSource(item.url ?? "").isMediaStoreUrl); + return { + outputs: [], + decision: await buildDecision( + "skipped", + selection.selected.map((item) => { + if (!nativeDeliverable(item)) { + return { attachmentIndex: item.index, attempts: [] }; + } + const attempt = { + type: "provider" as const, + provider: activeProvider, + model: model || undefined, + outcome: "skipped" as const, + reason, + }; + return { + attachmentIndex: item.index, + attempts: [attempt], + chosen: attempt, + }; + }), + { + ...buildDispositions({ kind: "handed-to-native-vision" }), + ...createAttachmentDispositions( + selection.selected.filter((item) => !nativeDeliverable(item)).map((item) => item.index), + { kind: "failed", reason: "remote-url image is not natively deliverable" }, + ), + }, + ), + }; } const entries = resolveModelEntries({ @@ -956,23 +1020,28 @@ export async function runCapability(params: { providerRegistry: params.providerRegistry, capability, activeModel: params.activeModel, + nativeVisionActive: capability === "image" && (await resolveNativeVisionFlag()) === true, }) ).map((entry) => ({ entry })); } if (resolvedEntries.length === 0) { return { outputs: [], - decision: { - capability, - outcome: "skipped", - attachments: selected.map((item) => ({ attachmentIndex: item.index, attempts: [] })), - }, + decision: await buildDecision( + "skipped", + selection.selected.map((item) => ({ + attachmentIndex: item.index, + attempts: [], + })), + buildDispositions({ kind: "no-model" }, { kind: "not-selected" }), + ), }; } const outputs: MediaUnderstandingOutput[] = []; const attachmentDecisions: MediaUnderstandingDecision["attachments"] = []; - for (const attachment of selected) { + const attachmentDispositions = buildDispositions({ kind: "failed" }, { kind: "not-selected" }); + for (const attachment of selection.selected) { const { output, attempts } = await runAttachmentEntries({ capability, cfg, @@ -989,22 +1058,22 @@ export async function runCapability(params: { if (output) { outputs.push(output); } + attachmentDispositions[attachment.index] = output ? { kind: "handled" } : { kind: "failed" }; attachmentDecisions.push({ attachmentIndex: attachment.index, attempts, chosen: attempts.find((attempt) => attempt.outcome === "success"), }); } - const decision: MediaUnderstandingDecision = { - capability, - outcome: - outputs.length > 0 - ? "success" - : hasFailedMediaAttempt(attachmentDecisions) - ? "failed" - : "skipped", - attachments: attachmentDecisions, - }; + const decision = await buildDecision( + outputs.length > 0 + ? "success" + : hasFailedMediaAttempt(attachmentDecisions) + ? "failed" + : "skipped", + attachmentDecisions, + attachmentDispositions, + ); if (decision.outcome === "failed") { logWarn(`media-understanding: ${formatDecisionSummary(decision)}`); } else if (shouldLogVerbose()) { diff --git a/src/media-understanding/runner.vision-skip.test.ts b/src/media-understanding/runner.vision-skip.test.ts index 7d3b8c5f069b..5d3c4e428058 100644 --- a/src/media-understanding/runner.vision-skip.test.ts +++ b/src/media-understanding/runner.vision-skip.test.ts @@ -187,6 +187,10 @@ describe("runCapability image skip", () => { let describeCalls = 0; const msgCtx = ctx as MsgContext; msgCtx.Body = "please inspect this image"; + msgCtx.media = Array.from({ length: 4 }, () => ({ + path: mediaPath, + contentType: "image/png", + })); const cfg = { agents: { defaults: { @@ -217,14 +221,171 @@ describe("runCapability image skip", () => { const attempt = imageDecision?.attachments[0]?.attempts[0]; expect(result.appliedImage).toBe(false); expect(imageDecision?.outcome).toBe("skipped"); + expect(imageDecision).toMatchObject({ nativeVisionActive: true }); expect(attempt?.outcome).toBe("skipped"); expect(attempt?.reason).toBe("primary model supports vision natively"); expect(describeCalls).toBe(0); expect(msgCtx.Body).not.toContain(plantedVisionSentinel); + expect(msgCtx.Body).not.toContain("Image attachment not"); }, ); }); + it("markers remote-url-only images instead of claiming native handoff", async () => { + await withMediaFixture( + { + filePrefix: "openclaw-image-url-only-no-handoff", + extension: "png", + mediaType: "image/png", + fileContents: Buffer.from("image"), + }, + async ({ ctx, mediaPath }) => { + const msgCtx = ctx as MsgContext; + msgCtx.Body = "please inspect both images"; + msgCtx.media = [ + { path: mediaPath, contentType: "image/png" }, + { url: "https://cdn.example.test/photos/second.png", contentType: "image/png" }, + ]; + + const result = await applyMediaUnderstanding({ + ctx: msgCtx, + cfg: { + tools: { media: { image: { attachments: { mode: "all", maxAttachments: 4 } } } }, + } as unknown as OpenClawConfig, + agentDir: "/tmp", + workspaceDir: path.dirname(mediaPath), + activeModel: { provider: "openai", model: "gpt-4.1" }, + }); + + const imageDecision = result.decisions.find((decision) => decision.capability === "image"); + expect(imageDecision?.outcome).toBe("skipped"); + expect(imageDecision?.attachmentDispositions).toMatchObject({ + 0: { kind: "handed-to-native-vision" }, + 1: { kind: "failed" }, + }); + // Local image stays suppressed (native hydration owns it); the + // remote-url image renders its failure despite nativeVisionActive. + expect(msgCtx.Body).toContain("[Image attachment could not be analyzed]"); + expect(msgCtx.Body).not.toContain("not processed"); + }, + ); + }); + + it("runs explicit image models untouched by native-vision probe failure", async () => { + await withMediaFixture( + { + filePrefix: "openclaw-image-explicit-model-probe-immune", + extension: "png", + mediaType: "image/png", + fileContents: Buffer.from("image"), + }, + async ({ ctx, mediaPath }) => { + const msgCtx = ctx as MsgContext; + msgCtx.Body = "inspect this image"; + msgCtx.media = [{ path: mediaPath, contentType: "image/png" }]; + const cfg = { + tools: { + media: { + models: [ + { + provider: "openrouter", + model: "google/gemini-2.5-flash", + capabilities: ["image"], + }, + ], + }, + }, + } as unknown as OpenClawConfig; + + await loadModelCatalog.withImplementation( + async () => { + throw new Error("catalog unavailable"); + }, + async () => { + const result = await applyMediaUnderstanding({ + ctx: msgCtx, + cfg, + agentDir: "/tmp", + workspaceDir: path.dirname(mediaPath), + providers: { + openrouter: { + id: "openrouter", + capabilities: ["image"], + describeImage: async (req) => ({ + text: plantedVisionSentinel, + model: req.model, + }), + }, + }, + activeModel: { provider: "openai", model: "gpt-4.1" }, + }); + + const imageDecision = result.decisions.find( + (decision) => decision.capability === "image", + ); + // The lone selected attachment leaves nothing to marker, so the + // probe never fires and catalog failure cannot reach this path. + expect(result.appliedImage).toBe(true); + expect(imageDecision?.outcome).toBe("success"); + expect(imageDecision?.attachmentDispositions).toEqual({ 0: { kind: "handled" } }); + expect(imageDecision).not.toHaveProperty("nativeVisionActive"); + }, + ); + }, + ); + }); + + it("keeps disabled outcomes precise and suppresses markers when the vision probe fails", async () => { + const ctx: MsgContext = { + Body: "inspect this image", + media: [{ path: "/tmp/image.png", contentType: "image/png" }], + }; + + await loadModelCatalog.withImplementation( + async () => { + throw new Error("catalog unavailable"); + }, + async () => { + const result = await applyMediaUnderstanding({ + ctx, + cfg: { tools: { media: { image: { enabled: false } } } }, + activeModel: { provider: "openai", model: "gpt-4.1" }, + }); + + const imageDecision = result.decisions.find((d) => d.capability === "image"); + expect(imageDecision).toMatchObject({ + outcome: "disabled", + attachmentDispositions: { 0: { kind: "capability-disabled" } }, + }); + // Probe failure leaves the flag unknown: no false delivery claim, no marker. + expect(imageDecision).not.toHaveProperty("nativeVisionActive"); + expect(ctx.Body).not.toContain("not analyzed"); + }, + ); + }); + + it("renders disabled markers when the active model has no native vision", async () => { + const ctx: MsgContext = { + Body: "inspect this image", + media: [{ path: "/tmp/image.png", contentType: "image/png" }], + }; + + const result = await applyMediaUnderstanding({ + ctx, + cfg: { tools: { media: { image: { enabled: false } } } }, + }); + + expect(result.decisions).toContainEqual( + expect.objectContaining({ + capability: "image", + outcome: "disabled", + nativeVisionActive: false, + attachmentDispositions: { 0: { kind: "capability-disabled" } }, + }), + ); + expect(ctx.Body).toContain("[Image attachment not analyzed: image understanding is disabled]"); + }); + it("skips agents.defaults.imageModel fallback when MiniMax M3 supports vision", async () => { catalog = [ ...baseCatalog, @@ -297,6 +458,10 @@ describe("runCapability image skip", () => { let describeCalls = 0; const msgCtx = ctx as MsgContext; msgCtx.Body = "please inspect this explicit image"; + msgCtx.media = Array.from({ length: 4 }, () => ({ + path: mediaPath, + contentType: "image/png", + })); const cfg = { tools: { media: { @@ -332,8 +497,17 @@ describe("runCapability image skip", () => { const imageDecision = result.decisions.find((decision) => decision.capability === "image"); expect(result.appliedImage).toBe(true); expect(imageDecision?.outcome).toBe("success"); + expect(imageDecision).toMatchObject({ + nativeVisionActive: true, + attachmentDispositions: { + 1: { kind: "not-selected" }, + 2: { kind: "not-selected" }, + 3: { kind: "not-selected" }, + }, + }); expect(describeCalls).toBe(1); expect(msgCtx.Body).toContain(plantedVisionSentinel); + expect(msgCtx.Body).not.toContain("attachment limit reached"); }, ); }); diff --git a/src/media-understanding/runtime.test.ts b/src/media-understanding/runtime.test.ts index d5ded12dc89b..9d3a290fdde3 100644 --- a/src/media-understanding/runtime.test.ts +++ b/src/media-understanding/runtime.test.ts @@ -135,7 +135,13 @@ describe("media-understanding runtime", () => { provider: undefined, model: undefined, output: undefined, - decision: { capability: "image", outcome: "disabled", attachments: [] }, + decision: { + capability: "image", + outcome: "disabled", + attachments: [], + attachmentDispositions: { 0: { kind: "capability-disabled" } }, + nativeVisionActive: false, + }, }); expect(mocks.buildProviderRegistry).not.toHaveBeenCalled(); diff --git a/src/media-understanding/runtime.ts b/src/media-understanding/runtime.ts index d7050e8f0cf6..7046127e0dbe 100644 --- a/src/media-understanding/runtime.ts +++ b/src/media-understanding/runtime.ts @@ -177,10 +177,19 @@ export async function runMediaUnderstandingFile( scopeContext: params.scopeContext, }); const attachments = normalizeMediaAttachments(ctx); + const decisionBase = { + capability: params.capability, + attachments: [], + ...(params.capability === "image" ? { nativeVisionActive: false } : {}), + }; if (attachments.length === 0) { return { text: undefined, - decision: { capability: params.capability, outcome: "no-attachment", attachments: [] }, + decision: { + ...decisionBase, + outcome: "no-attachment", + attachmentDispositions: {}, + }, }; } const config = cfg.tools?.media?.[params.capability]; @@ -190,10 +199,18 @@ export async function runMediaUnderstandingFile( provider: undefined, model: undefined, output: undefined, - decision: { capability: params.capability, outcome: "disabled", attachments: [] }, + decision: { + ...decisionBase, + outcome: "disabled", + attachmentDispositions: Object.fromEntries( + attachments.map((attachment) => [ + attachment.index, + { kind: "capability-disabled" as const }, + ]), + ), + }, }; } - const providerRegistry = buildProviderRegistry(undefined, cfg); const agentDir = params.agentDir ?? (params.agentId ? resolveAgentDir(cfg, params.agentId) : undefined); diff --git a/src/media-understanding/types.ts b/src/media-understanding/types.ts index e13344fbd354..2dc1cf3598fb 100644 --- a/src/media-understanding/types.ts +++ b/src/media-understanding/types.ts @@ -44,10 +44,25 @@ type MediaUnderstandingAttachmentDecision = { chosen?: MediaUnderstandingModelDecision; }; +export type MediaAttachmentDisposition = + | { kind: "handled" } + | { kind: "handed-to-native-vision" } + | { kind: "not-selected" } + | { kind: "capability-disabled" } + | { kind: "no-model" } + | { kind: "scope-denied" } + | { kind: "failed"; reason?: string }; + export type MediaUnderstandingDecision = { capability: MediaUnderstandingCapability; outcome: MediaUnderstandingDecisionOutcome; attachments: MediaUnderstandingAttachmentDecision[]; + // Optional on the shipped SDK contract: plugins pass FinalizedMsgContext into + // inbound-reply dispatch and may hold legacy decision literals. Core producers + // (runner, apply-capability, runtime) always populate it; absence renders no + // markers rather than breaking plugin compilation. + attachmentDispositions?: Record; + nativeVisionActive?: boolean; }; type MediaUnderstandingProviderRequestAuthOverride =