mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
refactor(agents)!: remove the session write lease (#121113)
* refactor(agents): remove session write lease * refactor(plugin-sdk): deprecate session write lease * refactor(doctor): remove session lock checks * test(agents): remove session lease fixtures * test(agents): align writer rebound assertion * refactor(infra): remove retired session lock exports * test(tooling): preserve embedded abort race shard
This commit is contained in:
Peter Steinberger
committed by
GitHub
GitHub
parent
67e653e580
commit
ce53f7e82e
@@ -1,3 +1,3 @@
|
||||
# Distinct OPENCLAW_* names in production source under src, packages, and extensions.
|
||||
# Ratchet: lower this number when cleanup removes names; never raise it.
|
||||
512
|
||||
507
|
||||
|
||||
@@ -3,22 +3,22 @@ a592581a61518734a8a410ec9f71069529ee5fbb54848daf650199eb1cb557ca module/account
|
||||
71522995185b956a0cc4927a472cc8d1153e5e998874bfd9a750513175174713 module/account-id
|
||||
64fc61ddb3ed2367ad193c8174e5ddf4f563d7456acd44ab3529219a8d0776fb module/account-resolution
|
||||
4fbb1c87e99399f842a20d75d5e35a4b7064a1b7f02115c23f9a2a7cdcfb57ee module/agent-config-primitives
|
||||
f989bfd75d607f1405ada99d3b7cf8407119187a850d264c99e8d5bb0d52e0b6 module/agent-harness
|
||||
58a05367810e535955fdfdcd21b79c97b1f188dbb3d0f87265b1e6c831926bd6 module/agent-harness-runtime
|
||||
a9c706069df02e5a876bbf3e35ef55e9eb28906ab84f4f099896ca01ffe15ee7 module/agent-harness
|
||||
a1af686024edde774ff921f36f398b49021e3acfa21c6b2d943aafda44400ae6 module/agent-harness-runtime
|
||||
762a2c3df44621e9464fdd3ef437be2a878bfa9a7e8ca7670f88ae549e601614 module/agent-media-payload
|
||||
587c6f9abe54d1a309b9a720c5bab52b69b30e396765a487982f5e587d509f96 module/agent-runtime
|
||||
c2ed16c7b059e188e15511fb843fa3778e6469165f74d0de35b775c9760b7b37 module/agent-runtime
|
||||
83c954ae767298961748011282c5f79d5cd7a4187f8769a91e85cad7ee950902 module/agent-scope-runtime
|
||||
8fecb210e22bce4532b6ab649b09465f0bd2c857a44abf40db7d683d6491e6da module/allow-from
|
||||
f0cd21c1854e209363fc2cbc684ba9dee9668ba7b5d0b57a96d4c8da825d7337 module/allowlist-config-edit
|
||||
8b20c8157e04f13233c239401185ae23f52565a7dd337274f954b30e890be6a6 module/approval-auth-runtime
|
||||
0ae73ab9450fb773515a5247505d18706193fb0670654e3af55f03400ed0a0a8 module/approval-client-runtime
|
||||
50b21c297db54c391689ba805b5ffd1620e7d0891cae98bed9f6c41929690988 module/approval-delivery-runtime
|
||||
131f830c475cec1adb2f8a0904b7f750411f87d0c76b110ca528125ee93f52a1 module/approval-auth-runtime
|
||||
10b7388a85818918433239e11afe82426cd37f2fffb96f7711116e4e7ed22fe5 module/approval-client-runtime
|
||||
d8872a2e0579c618ab27bfca31581eef9c0d12c6e39f07c951b1921cb57f2ab7 module/approval-delivery-runtime
|
||||
afba69ce95b3a939511c1f1e6b27a5999e8793304cba7578b0785c7edc342580 module/approval-gateway-runtime
|
||||
a9f10f7c70287d7dc8ad224ca6424234652e74908ae7de2fc7edf4c98c91de4a module/approval-handler-adapter-runtime
|
||||
f8b5c31c956b567827d58e9a1f6493846448b3c0ae30904cc7f4d872b8071d26 module/approval-handler-runtime
|
||||
97f96bde0c23596a1d25e43bb2145d20de9b93b4657ac2ebdf7248be086b05da module/approval-native-runtime
|
||||
85a5c33e5ce2137b469b4d432274185a49fa5146421e0985a5c837447ef1697c module/approval-native-runtime
|
||||
b102781a2d78bbfafcf4205aeb2c169ad69bc4010edaf376647036cf3fba48d7 module/approval-reply-runtime
|
||||
031443b5f1f815eb9c17d1bc808bd8737ae4a0493cca8bd170b7ef2c5c3426dc module/approval-runtime
|
||||
eff194a7ab6fbdcebb021633645fd5df49b6744f6b1297cc78404f396aea4b14 module/approval-runtime
|
||||
01ca912836b8dec672f705e294f72d346e778557e4c591317d67558ea7669c0b module/archive
|
||||
d7e53de63b0ac11a266e4abdc18ba6e9401b80309f5c8f5f6a72a00f65dfe3bd module/boolean-param
|
||||
076689cd8e62c842580d61a34d97bcf42c90406287d428fc5d0d3eb373ec274d module/channel-actions
|
||||
@@ -26,46 +26,46 @@ d1341161e2f5d44eaf7eca3a777ab3f106c7db1db32778bf98c75c7fa7d57ca4 module/channel
|
||||
c2cc71d5070b6071c51248b0648d1ad1a9468d3737df890adc77ec02025e8853 module/channel-config-primitives
|
||||
9fa7e5f9b3515f0bbc252210b034d39c25de9eb431def30dddce3198c309efac module/channel-config-schema
|
||||
a6af4971aa36345f1a32d97981bd36cc29794f62f30bd57859df2816c65f6fde module/channel-contract
|
||||
3ce6bdb43ae6af29630c1a7f5db5401cd3f0a757d8c957ef8bb451eb463d052b module/channel-core
|
||||
68ad8c35e936c03801b562ecdce87650c0ed45cff02b2c87dd1c952164553984 module/channel-core
|
||||
40f8e726de245e17cf7bf908bb7d6fb084f4927f5ccc20b0c089f47af1bd7a0e module/channel-dm-policy
|
||||
87c2a2117a21eaca7e90399a59a7f4177d72061ca47e114b9fbf679e00f5ae39 module/channel-entry-contract
|
||||
f1013a2efe1df0df5f3c6916bd036e8fdef60edb4d1d9978ced12b27c60f84a8 module/channel-entry-contract
|
||||
56b185eb99eb4981056befd38cd9511b655923934acb1c22aeec7d9c504bdf96 module/channel-feedback
|
||||
5cb907889536199b49c587b87b296b37e32dcd0710ee847a460aa2442ee8ec9b module/channel-inbound
|
||||
1fd4109d3e6fcefb529268236fce17dccf73c1e3ed9f8bbccff40e2a4a008a5c module/channel-inbound
|
||||
780abefd1600c55da219da43c769c873d071ab616b5e02d22f365677ed5a7be4 module/channel-inbound-debounce
|
||||
b72485da4fe5d7f73352f4474d00e30b749b64c8a2066755e104958cd96036c2 module/channel-ingress-runtime
|
||||
f83762680c0d0aa6fa1bad9e53c70bfdf3694126dbdd9491db764a69ed3f6d67 module/channel-lifecycle
|
||||
0e47457e38d1df0bd572e1408cde2ca6a788b65205f43c585316b5ad3a8f2f16 module/channel-logging
|
||||
fab761166333914424d58c6cbfd278dbae11ebe85715a107e6885ca3a25831d9 module/channel-message
|
||||
ee87fc2421c43535e8362a81311c658c87a79820d0b99096bc6076435360132b module/channel-outbound
|
||||
0fca740b839b297b29e19782feb4347523a8360e9496f90abebd41020fb76149 module/channel-pairing
|
||||
0cf2c3ab1cb5520099620040215312148211b77dd16d514082caeff52f4f6855 module/channel-plugin-common
|
||||
a72e82470c1b985275f78a14dc76f6c8b6a1ec3a25a19160ba2653d2f779984d module/channel-message
|
||||
23944f54612c09665ee4f32b42ae83228b560edb8f795176e66f482d8c5236cd module/channel-outbound
|
||||
32796d1a1437955d4b20ae474205bda1a5f3e6d87b01d3abe69d48075b83c88f module/channel-pairing
|
||||
bac1a249dc3b2450b4e8cda1f6782d89a4224587525a48d46361204964b2f4e7 module/channel-plugin-common
|
||||
c2c7900e47c0595bbc8a166aa60cfbc5a0e02f9a4c0985d5fd630c7dec4c3dca module/channel-policy
|
||||
e787a315cec9681a49a9e5d1af8fe4fae345561d30996888e5ef6593faae0a75 module/channel-reply-pipeline
|
||||
1a142544f8c8e2bbf250bc582949473ca1f72c9d7c0b917b8c8bb6af38bc8f7f module/channel-reply-pipeline
|
||||
482370e60135db9bfaf07f24bab549e5fde09ab265a6061a1f587c5d93929e91 module/channel-runtime-context
|
||||
32c2e9ea4f954a6b04d6e61b392e63d42b5cc74ccbdc1404c94a4c97463f0ca1 module/channel-secret-basic-runtime
|
||||
941673d941b3ab9bd553ba9153305af7933019254a504a603907a237d06b5e79 module/channel-secret-runtime
|
||||
7f5bc35b328c716022a442804f3e83416ee23eaddb49e771c385660dc6540612 module/channel-send-result
|
||||
32b446e32f9b9b54008e81ac7069869098c14ca6620e5ca700e41a0544d7b614 module/channel-send-result
|
||||
4493cf54a54d7159e27d33884b1cd231cd54a038a85a3d5d3676e149954444a7 module/channel-setup
|
||||
66df387345c0d64083904f4271228bb1e5660e40b00f42afc5d802d3ff07627f module/channel-status
|
||||
198ed5042f86da7731dad61bc66f44bcb249fd9e369d79cf4509df172d5cab7f module/channel-streaming
|
||||
67df67da5ae72e9eaeb19d41b6bd2432ec4fd8b7b63b2b616fb98f3b4e0ec41d module/channel-streaming-config
|
||||
fdeffe356c7c4edeec9f8fd03edcadc375eabc7a9412e582b10c3180e3ef40fc module/cli-argv
|
||||
ad12670dbfe538f8d0ebf4fb2b68080e93a760278278e6b1ce9bb129d4b2d533 module/collection-runtime
|
||||
8f19b5729eb9e38ade635a3759d307826fb00ea5cf8d4b50343b54511dfe6e01 module/command-auth
|
||||
7126e3cec538129822aebdb2fba88de11bee5fb689bff3120a8e25bfc697d603 module/command-auth-native
|
||||
b705cd511e3a1ad8da44b0459ffb57bcf21482dfa6eb0016a6db99dc52cfb89d module/command-auth
|
||||
3bcb143c3e1de4366db7981f8ba18c7cf809437b049cb6025f750b360da9c851 module/command-auth-native
|
||||
396f60be14d8fe1144076cffeafc2e2ddc0379b5e664f9fd45d13a0edc4a9f2f module/command-detection
|
||||
841389ea61d1389d1dbcc9350832e5cb5307ab5b380cbc14800ea62796e06ed7 module/command-primitives-runtime
|
||||
0ab02df5c3904386bfb062e7dce0a24e2ed835b54dcc0112e367346624cb0efe module/command-status
|
||||
02011f70a5de5f1861609206038b101d3e9bbf39bf45a3cfaaae56596299225b module/config-contracts
|
||||
7ee5bc35d30165be33ab229a96dd6a6e3e04c620fa4c41753d568798c514feb3 module/config-mutation
|
||||
eac10f08466e8513c5b5663b29ca3d4ca76d66858367163c5239f08f33d7024c module/config-runtime
|
||||
2d4bcc06be68852dd829d47b295e5318a99ccd4db6c8c14bf4742c6b58a517a2 module/config-runtime
|
||||
069ca892c9d9c0d82f9a0d1e72970da2b1bc5d3c4fc15e127b18bbf78292cd2a module/conversation-runtime
|
||||
e028a493e9d30afbd1d58ee28539d9dd47d160065ad44ccdb3473badd7a71550 module/core
|
||||
be2e2ce71478e859d9b6d68d32d31a48c6591d0065545bc45a1c4db8e684b30e module/dedupe-runtime
|
||||
3b7142dc38efc70b115cfc84d291ba3e848c2d385ae9a62fd93615e9f3c7a561 module/core
|
||||
dbff5cec0161ca3afcabda96be29b7c9832f37eacc88ea4f5c7389ae6db263f2 module/dedupe-runtime
|
||||
ebef0e650ab45e44c9335e2b3e15588c968cea6dadd125364a076f9c50ad1e8c module/device-bootstrap
|
||||
9ecd22bc3acae3f9b0556b305c4232abe54a7993b910caa789cba6cf917dd1e2 module/diagnostic-runtime
|
||||
6d90412b97efcc675d16acc3aeb2752520a613ce4bb25b87e47f84fd6b36736d module/directory-runtime
|
||||
c2c20568f733021911438f66efa21acd013f458dd44f33f7df5530084eeddb8d module/discord
|
||||
aa2d26865800cc400860be02edbcc2fba4f2a7448b44bb10540abf03c197cd15 module/discord
|
||||
f2d69888d0c799e12d97b5f92393906aa843be25f61cb6a6c55d32f1db800a44 module/error-runtime
|
||||
1caf8cc5552ca5e392599457e3f9616e033de199d9f47c12320cdee617a6b0e9 module/extension-shared
|
||||
dd9f6e0fd33cc88b22543c1ee30cc09cf4de4d8f30dff7b7f9cebef885c21543 module/gateway-method-runtime
|
||||
@@ -75,8 +75,8 @@ f558a90b52bda7e431cc46005f8735263ee065942925f9d547ae528f218be479 module/gateway
|
||||
53e032e2f3a2f434702ce1633b0d2a0bed5cbe5f6ef01d7dc348967395036b9a module/hook-runtime
|
||||
4d9f7d3c3e59113b493f3cac812b3c117968035af10a751a92f2aca8e2ec8806 module/inbound-envelope
|
||||
4928af5d2509f696b896f53ac790303a0742202dbcdae3e44fe6d1b434a9c1ba module/inbound-event-delivery
|
||||
ad782ab0200a135c6c7d6ce6fd2079bf46c6834adb2d984e527559c1b3ab608c module/inbound-reply-dispatch
|
||||
97a718b5c0f276c7cb0feab83e5aa2bb0f9a14175236d94d4c9f7a7baeb05137 module/infra-runtime
|
||||
1f3da2dc90e3e02b81383d367620a9f65ac299ff1933bc908e388e9dd7fa2884 module/inbound-reply-dispatch
|
||||
660435bdefd6371816fcb1676096a5840ce78347f9788eee3e50c08f492ff5a8 module/infra-runtime
|
||||
ce73721421f1b903dd04ead4df173582e59ea3e9990248102c448b419cc6d272 module/ingress-effect-once
|
||||
2fdc37835ccc4651c580847250adf8e42c165483eb239907aea981a6552e24b4 module/interactive-runtime
|
||||
408d257ab5cc4b88a22b7e7595039cb8fc524b261c44141b294fbd0100ba62ee module/json-store
|
||||
@@ -85,32 +85,32 @@ e5acf130491ef188b193a3e3dc5284e53a29f7e08278caa188f5ddb96e860dc4 module/logging
|
||||
f1ca4ced4305d0769c2d8cc1291137ac7002fe0e6eaec2c1a71edad2204c8311 module/matrix
|
||||
8cef237e683b323062fa4134a88a0b5b8ff97f1c5d356654c8062fdbf5eed87c module/media-local-roots
|
||||
f74d7295fe716aa140aa0bc9300d6259d71dab826de0808fca6bb02592bf5d6e module/media-mime
|
||||
d0c103832eac668fbbd1593c1c9e3ee77b4233f4b336a6d94cd2f1b032c3050f module/media-runtime
|
||||
1cd61c7f1721e2af43cc307c5d9cb03f072bf5666aa1e3906d322a9ec038406d module/media-runtime
|
||||
6a52f93107335f88751704352cc01e62add06f854a5b7d765e2a5ee87c0313b6 module/media-store
|
||||
6e5b09a983a69041873ece5bc3b719c0dc913e5a69e2afdf6c7958889e711097 module/media-understanding
|
||||
7f74e947cca0cf894869bede29815863d5eacca016d04d6262e9d41b3999ae94 module/media-understanding-runtime
|
||||
9d94b6190c4a8f0fc508c1bb17339d16e943feb03bdb9ada7b25b67c345607a1 module/meeting-runtime
|
||||
0a63357429e02a239cfcd6f8ccedd50f254fa9f78c4e4fef8a014b8808011d2f module/meeting-runtime
|
||||
e5d2b540090c17602a1c36b42559f7b516e20730e06bbdd8606597c66ef2ef95 module/memory-core-host-engine-foundation
|
||||
38d54e0fcd07f475fccfdb3cff87808dfa4a7e54ed6d35cd61cfc81cae11db53 module/memory-host-core
|
||||
aafaf500f6593e6e907ab970d1bea7aff00dd7499c78dac2ea6f73fc4d7f595d module/memory-host-core
|
||||
1efa0aadc4261d1c6073058cbf3dcc9fa681424819bdd14333e19b249bbc4b18 module/messaging-targets
|
||||
577d82ef09e96de6af83a22c9b89cc1ddf2d76964abac5808b102ce04ced294f module/model-session-runtime
|
||||
4f6572822a931393b83044bf8a5dd792376a8abc603a8bf175a98cdbbdf66390 module/models-provider-runtime
|
||||
1c30f86bcb65bf462005a90aa78efc170e98563d960cf29184c5253fd6e8a9b8 module/model-session-runtime
|
||||
dc9dc9645c3a1e1d3bf35538fef0e6ccf6e4df27b8ae937783683f2af16f6495 module/models-provider-runtime
|
||||
b518b4caa7f138d448fe08f5fc9e5f67da5d4b26aafaee2370feb2300e1baa21 module/native-command-config-runtime
|
||||
7a5b56c37e5d862feab8258eb9f79f44e6946333c93b0c22dd3f792b33a7c19a module/native-command-registry
|
||||
15d1d490e4f7909d2d563b83988258a9cc5fa1344cffb0cd3dcd7592c0467632 module/param-readers
|
||||
ca7a56bb1a6169b4cf9befbf5aa21da280a8086fdc49fca4eec520a7a7c98549 module/persistent-dedupe
|
||||
f8b250da3eab2f85b2bf358872f910b29e541ea04358498a6deab353fa0520fc module/plugin-config-runtime
|
||||
7a8cfa7161563aad7450c0a1c43317b4dfec5c3b0807b4c66aa923ded3cf0a1d module/plugin-entry
|
||||
282ab9d7de884bda645c3b40fd19a30d2061d7efc437226f469fa657ee15d99a module/plugin-runtime
|
||||
37486f578a06306f9bcbb980245fa032f0a8e18daff4fdb24f0852409304c1a2 module/provider-auth
|
||||
1b831d62daf65ffcb48de16cad3e065eae0a4318776c26f3cb10c09016b1ac62 module/provider-catalog-runtime
|
||||
a7a2e4977e8ef784fa2852c047160cc4c3b3b751597ebc74e567bb47ae4de8d1 module/plugin-entry
|
||||
0cffe85023a15806f23db7bd1c9f255d1a24653620e47d0b790ad99b5d4095c7 module/plugin-runtime
|
||||
e195c5cc5682b3d37ff64e3fb3895d47ba5cccacf7be46ca22f5634d85dc48a7 module/provider-auth
|
||||
f1e3c0fa27eb8cc6b7252c30315001347e12709d0587a172e00f230247637f2c module/provider-catalog-runtime
|
||||
8131147d699394bd06503e2ea2f5f1a50b1594a87dded6d118b74a8d0328c8f6 module/proxy-capture
|
||||
913c32adcd94ea9c8b7efac0f7c65231500c49129e638c0682eb632da3d5a0c8 module/question-gateway-runtime
|
||||
f319c52a0f1655de752efa55a983893e56e4afa0727f6cc4896c3238c984e91c module/reply-chunking
|
||||
2b3caf36f524ef7cac9caecadcc394fc333c8f122ee9c8bb8c99a688d290f2f4 module/reply-dispatch-runtime
|
||||
9903ca6e9eedc144e3bce91ec74cddeb67a7af109966094cc12c2f1cacdb03c4 module/reply-dispatch-runtime
|
||||
73f861fa3179d5af1159853c5acab0eec7a6c8f9398dcb75ea770e784fca6727 module/reply-history
|
||||
ecc6e3190892ff416ce69d26c8ce0aa08a468c8689ca3606c18b0271e2d87597 module/reply-payload
|
||||
dcef2112bc3a57b70306a800b0420a44e2b02d1f6ca47d13c8508c78b5bccf37 module/reply-runtime
|
||||
405614f15ef5cde172c08d8f85917a22162ddb4ad6ebffafe91343131028f8cd module/reply-runtime
|
||||
aa07d85d99fdd2b1e0cbe9975fb6dcae66b8bdce2607c6bd5402ae68bb15118c module/root-walk
|
||||
292405cda407e2569f731212f6f981b12bfe682836d5f6e58c6ae194ba27f144 module/routing
|
||||
7877a7e58fa32a64107154e5b714c6d165e96989d4aa5f43e0afac085a187af0 module/run-command
|
||||
@@ -118,20 +118,20 @@ aa07d85d99fdd2b1e0cbe9975fb6dcae66b8bdce2607c6bd5402ae68bb15118c module/root-wa
|
||||
f7900c5ff5827e1b53a926367d807bd4c81b313ab74fd0409d571ae881226a60 module/runtime-config-snapshot
|
||||
8e33ccbbe610c6c3ad04a140f7511a16cf406733c8459216b1fb4b1e7f0208d9 module/runtime-env
|
||||
7e871b7319745678bb83fcfc1b54c8751b0ab1af92ff06c01d0659ac92863c11 module/runtime-group-policy
|
||||
8e2bef59679a87201f38367f84ee2da8c2c055c6da4444f5ab52f5a7be49e7de module/runtime-store
|
||||
f03c34763b71ed4a1e512295191d5e0f2cb289fbf8adbfa4b6053977ba7adef6 module/runtime-store
|
||||
d17862c40825af1ddf0257b44f1e1cbb9c375e8e5ed668fae75d530d1a465cf9 module/secret-file
|
||||
8e2ac4d3973d8d8ce4478e3440d66ee5c0d9213b0fe9e927c421d14fd31e5e86 module/secret-input
|
||||
5958846179128ea6b29135fb90ce4e40b8df7de63d1c65d1696e52294cbc0f52 module/secret-input-runtime
|
||||
72ca4634e0038ad6dc80c4ce12eb34014e1fa990c7eabe19bb4501b2722ee685 module/secret-ref-runtime
|
||||
6db2af355b0d1d46f493a5d34a63a6ea5231dafbd8b896d976a52da681868f67 module/security-runtime
|
||||
cfdecb8a5744c5f8acdde768e41a5c370df0a48325c8956f0827db4b446cc1db module/session-catalog
|
||||
b2dc6360d27155152d4415a78c62dcc91a8a46321f9fbcb02ebb006668bbd0cd module/session-discussion
|
||||
bbfe5fe057d23fd874689776aff2d9aa506877c683e275ee66fa28fd866fc58e module/session-store-runtime
|
||||
e0b0dd5f31394d5bba70608a0f31f2738c1a2f338cbf92feabf620ba18f8e835 module/session-catalog
|
||||
db4b4610a071ae2f2a456f11fc2a03ecf0554b946138b46c51cb7badca7ec173 module/session-discussion
|
||||
6df070e65e9eda3ffa42894482a20437ae4932aba97c22b469e3ca8eeb9f74fa module/session-store-runtime
|
||||
e54f5c86a55683028a8c4a7c821cf9b9f11b48f91ae9b08e79f8e94b91eb5fde module/setup
|
||||
41c4790efda179c600fcc8cc4f2cd9c0505efd86df15a1a840aeb99c839bc627 module/setup-runtime
|
||||
44d37e0d9131ad2859f41068f2604090c784e65f1bd6ebda8e051b6f2e5e1660 module/setup-tools
|
||||
9c36f9dd90bee4b1fd51812862d835b54377bfbe4b9b9387d33ed35863eb924b module/skill-commands-runtime
|
||||
04b995526eb91e230b5334e4f4e119e93423d480d4bd72b9c186abd198cef2ca module/speech-settings
|
||||
ba8b411439e210f01f95322554e49ce82abcbe25ba968746feeffbca8af285a4 module/skill-commands-runtime
|
||||
f11732d0be82559c2a234dbf62513a2a9bd12bd20703cfb6258da620f3b9cb05 module/speech-settings
|
||||
1e4df0f48b50a8eb546459af1e947ba143c287e3d9c13706c45b67d411c46ad5 module/ssrf-policy
|
||||
e4940099376e18e34c234d4d90d836cbe234c3e891e40b7b4879168b386b1121 module/ssrf-runtime
|
||||
3c3c812d2d0c997ec81d117346347b802a3445e8e0d76aa6d0e3259f3cf55f98 module/state-paths
|
||||
@@ -141,11 +141,11 @@ f097d0096b21c8a052f0f649b7512ecf2aba4744ae6956f001950e053828b309 module/string-
|
||||
aef35bee2502cd6ed8765409b758e452aff8ac9469fd773e6a2a44c9a1bc3f66 module/temp-path
|
||||
87fa81b9e58d8fc04a4b4202d2d37fca339615f5225687d9db905151439e0f4d module/text-chunking
|
||||
50d2fa56b2ad57edd12d74201d18ab43045b5c9da0ab98cc158cebe9bd8b768b module/text-runtime
|
||||
b81beab0301b6d44e366b70376f914d630bd09c1428fdd2cb6ec869f8c2736e3 module/tool-plugin
|
||||
c850a6ad69309b2a8e279f7c8f44bce67cdf14aeba89ebf5064337a5e41f8816 module/tool-plugin
|
||||
dc1a073c59ab61e2789533b777b3f0cb9af689d64a97796b10e8aa82552510db module/tool-results
|
||||
5d4f1dbde1238799e5244b9c0b799aa2fb7d607d0c072d9a70659978825ef4ee module/tool-send
|
||||
cda105b721d498df23a554c6b68be150b8fe66b8b9172185c31a0b3b0646b1dc module/web-media
|
||||
3699bee6cba14d964d380e02cb7e17817abaeb4e238e22b080afeac54dbc5065 module/webhook-ingress
|
||||
020fbccf78933eddcab3175480455b03289bc6ddd4f2e1cd97bbc88186bbc3f7 module/webhook-ingress
|
||||
ddd05cb74246c393fef4d3fd8331eddd4db72a000fe45e9e515357b757636854 module/webhook-request-guards
|
||||
de59e86e126b75d13251cba7ebbe27b44d9b5588785d98df5ff4d6722374c81f module/widget-html
|
||||
9161b36ec0ab062ea41b363c894fcd672a7727f21cb726739f99f9c184fce69d module/zod
|
||||
|
||||
@@ -190,7 +190,6 @@ Modernized core doctor checks stay attached to the ordered doctor contribution t
|
||||
```bash
|
||||
openclaw doctor --lint --only core/doctor/gateway-config --json
|
||||
openclaw doctor --lint --skip core/doctor/skills-readiness
|
||||
openclaw doctor --lint --all --skip core/doctor/session-locks
|
||||
```
|
||||
|
||||
`--only` and `--skip` accept full check ids and may be repeated. If an `--only` id is not registered, no check runs for that id; use `checksRun`/`checksSkipped` in the output to confirm a focused gate selects the checks you expect.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
summary: "Agent loop lifecycle, streams, and wait semantics"
|
||||
read_when:
|
||||
- You need an exact walkthrough of the agent loop or lifecycle events
|
||||
- You are changing session queueing, transcript writes, or session write lock behavior
|
||||
- You are changing session queueing, writer claims, or transcript write fencing
|
||||
title: "Agent loop"
|
||||
---
|
||||
|
||||
@@ -27,14 +27,14 @@ execution, streaming, persistence.
|
||||
|
||||
Runs are serialized per session key (session lane) and optionally through a global lane, preventing tool/session races. Messaging channels choose a queue mode (steer/followup/collect/interrupt) that feeds this lane system; see [Command Queue](/concepts/queue).
|
||||
|
||||
Transcript writes are serialized by the per-session lane and the SQLite writer queue. Each append or rewrite validates the current session identity inside its synchronous commit transaction, so a stale run cannot overwrite a newer session generation.
|
||||
Before streaming, an admitted run records its durable `activeWriterRunId` claim. Every transcript append or rewrite supplies `expectedWriterRunId`, and the synchronous commit transaction verifies that it still matches the active claim. A superseded run therefore cannot commit stale transcript data. The SQLite writer queue orders per-agent mutations, while the Gateway state-directory lock prevents another Gateway or `openclaw agent --local` process from owning the same state directory concurrently.
|
||||
|
||||
## Session and workspace preparation
|
||||
|
||||
- Workspace is resolved and created; sandboxed runs may redirect to a sandbox workspace root.
|
||||
- Skills are loaded (or reused from a snapshot) and injected into env and prompt.
|
||||
- Bootstrap/context files are resolved and injected into the system prompt.
|
||||
- A session write lock is acquired and the session transcript target is prepared before streaming starts. Any later transcript rewrite, compaction, or truncation path must take the same lock before mutating the SQLite transcript rows.
|
||||
- The session transcript target and writer claim are prepared before streaming starts. Later rewrites, compaction, and truncation use the same in-transaction writer-claim fence.
|
||||
|
||||
## Prompt assembly
|
||||
|
||||
|
||||
@@ -222,6 +222,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It
|
||||
| `plugin-sdk/runtime-env` | Narrow runtime env, logger, timeout, retry, and backoff helpers |
|
||||
| `plugin-sdk/browser-config` | Private-local after July 2026; Supported browser config facade for normalized profile/defaults, CDP URL parsing, and browser-control auth helpers |
|
||||
| `plugin-sdk/agent-harness-task-runtime` | Private-local after July 2026; Generic task lifecycle and completion delivery helpers for harness-backed agents using a host-issued task scope |
|
||||
| `plugin-sdk/agent-harness-runtime` | Agent-harness runtime helpers. `acquireSessionWriteLock`, `resolveSessionWriteLockAcquireTimeoutMs`, `resolveSessionWriteLockOptions`, and `SessionWriteLockAcquireTimeoutConfig` are deprecated no-op compatibility exports scheduled for removal in the 2026.10 release train. They no longer block or create lock sidecars; harnesses should rely on OpenClaw's per-session lane plus the durable writer claim and in-transaction fence. |
|
||||
| `plugin-sdk/codex-mcp-projection` | Private-local after July 2026; Reserved bundled Codex helper for projecting user MCP server config into Codex thread config; not for third-party plugins |
|
||||
| `plugin-sdk/codex-session-transcript-runtime` | Private-local bundled Codex helper for serializing transcript-mirror writes; not for third-party plugins |
|
||||
| `plugin-sdk/channel-runtime-context` | Generic channel runtime-context registration and lookup helpers |
|
||||
|
||||
@@ -76,11 +76,9 @@ Normal Gateway writes flow through the session accessor, which serializes per-ag
|
||||
|
||||
OpenClaw no longer creates automatic `sessions.json.bak.*` rotation backups during Gateway writes. The current schema rejects the legacy `session.maintenance.rotateBytes` key, and `openclaw doctor --fix` removes it from older configs.
|
||||
|
||||
Transcript mutations use the session write queue for the SQLite transcript target:
|
||||
|
||||
Session write locks use fixed production defaults. The corresponding
|
||||
`OPENCLAW_SESSION_WRITE_LOCK_*` environment variables remain available for
|
||||
process-level diagnostics and emergency overrides.
|
||||
Transcript mutations pass through the session accessor and SQLite writer queue.
|
||||
Each mutation verifies the active run's durable writer claim inside its commit
|
||||
transaction, so a superseded run cannot write to the transcript.
|
||||
|
||||
### Downgrading After The SQLite Flip
|
||||
|
||||
|
||||
@@ -96,52 +96,6 @@ describe("qa suite runtime agent session helpers", () => {
|
||||
expect(options?.timeoutMs).toBe(60_000);
|
||||
});
|
||||
|
||||
it("retries transient session store lock timeouts while creating sessions", async () => {
|
||||
const lockTimeoutError = Object.assign(
|
||||
new Error("SessionWriteLockTimeoutError: session file locked"),
|
||||
{ code: "OPENCLAW_SESSION_WRITE_LOCK_TIMEOUT" },
|
||||
);
|
||||
gatewayCall
|
||||
.mockRejectedValueOnce(lockTimeoutError)
|
||||
.mockResolvedValueOnce({ key: " session-2 " });
|
||||
|
||||
vi.useFakeTimers();
|
||||
const pending = createSession(env, "Retry Session", "agent:qa:retry");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
|
||||
await expect(pending).resolves.toBe("session-2");
|
||||
expect(gatewayCall).toHaveBeenCalledTimes(2);
|
||||
expect(gatewayCall).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"sessions.create",
|
||||
{ label: "Retry Session", key: "agent:qa:retry" },
|
||||
expect.objectContaining({ timeoutMs: expect.any(Number) }),
|
||||
);
|
||||
});
|
||||
|
||||
it("retries transient session store stale locks while creating sessions", async () => {
|
||||
const lockStaleError = Object.assign(
|
||||
new Error("SessionWriteLockStaleError: session file lock stale"),
|
||||
{ code: "OPENCLAW_SESSION_WRITE_LOCK_STALE" },
|
||||
);
|
||||
gatewayCall.mockRejectedValueOnce(lockStaleError).mockResolvedValueOnce({ key: " session-3 " });
|
||||
|
||||
vi.useFakeTimers();
|
||||
const pending = createSession(env, "Retry Stale Session", "agent:qa:stale-retry");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
|
||||
await expect(pending).resolves.toBe("session-3");
|
||||
expect(gatewayCall).toHaveBeenCalledTimes(2);
|
||||
expect(gatewayCall).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
"sessions.create",
|
||||
{ label: "Retry Stale Session", key: "agent:qa:stale-retry" },
|
||||
expect.objectContaining({ timeoutMs: expect.any(Number) }),
|
||||
);
|
||||
});
|
||||
|
||||
it("reads effective tool ids once and drops blanks", async () => {
|
||||
gatewayCall.mockResolvedValueOnce({
|
||||
groups: [
|
||||
|
||||
@@ -41,7 +41,6 @@ type QaSessionTranscriptSeedParams = {
|
||||
updatedAt: number;
|
||||
};
|
||||
|
||||
const SESSION_STORE_LOCK_RETRY_DELAYS_MS = [1_000, 3_000, 5_000] as const;
|
||||
const SESSION_STORE_FTS_SETTLE_RETRY_DELAYS_MS = [100, 250, 500, 1_000, 2_000] as const;
|
||||
const MAX_COMPACTION_SUMMARIES = 16;
|
||||
const MAX_SUCCESSFUL_TOOL_CALL_EVENTS = 64;
|
||||
@@ -69,18 +68,6 @@ type QaSessionTranscriptSummaryOptions = {
|
||||
allowEmpty?: boolean;
|
||||
};
|
||||
|
||||
function isSessionStoreLockTimeout(error: unknown) {
|
||||
const text = formatErrorMessage(error);
|
||||
return (
|
||||
text.includes("OPENCLAW_SESSION_WRITE_LOCK_TIMEOUT") ||
|
||||
text.includes("OPENCLAW_SESSION_WRITE_LOCK_STALE") ||
|
||||
text.includes("SessionWriteLockTimeoutError") ||
|
||||
text.includes("SessionWriteLockStaleError") ||
|
||||
text.includes("session file locked") ||
|
||||
text.includes("session file lock stale")
|
||||
);
|
||||
}
|
||||
|
||||
function isSessionStoreFtsSettleRace(error: unknown) {
|
||||
const text = formatErrorMessage(error);
|
||||
return (
|
||||
@@ -261,29 +248,8 @@ function emptySessionTranscriptSummary(eventCursor: number): QaSessionTranscript
|
||||
};
|
||||
}
|
||||
|
||||
async function callGatewayWithSessionStoreLockRetry<T>(
|
||||
env: QaGatewayCallEnv,
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
options: { timeoutMs: number },
|
||||
) {
|
||||
const retryDelaysMs = SESSION_STORE_LOCK_RETRY_DELAYS_MS;
|
||||
for (let attempt = 0; attempt <= retryDelaysMs.length; attempt += 1) {
|
||||
try {
|
||||
return (await env.gateway.call(method, params, options)) as T;
|
||||
} catch (error) {
|
||||
if (!isSessionStoreLockTimeout(error) || attempt === retryDelaysMs.length) {
|
||||
throw error;
|
||||
}
|
||||
await sleep(retryDelaysMs[attempt]);
|
||||
}
|
||||
}
|
||||
throw new Error(`${method} failed after session store lock retries`);
|
||||
}
|
||||
|
||||
async function createSession(env: QaGatewayCallEnv, label: string, key?: string) {
|
||||
const created = await callGatewayWithSessionStoreLockRetry<{ key?: string }>(
|
||||
env,
|
||||
const created = (await env.gateway.call(
|
||||
"sessions.create",
|
||||
{
|
||||
label,
|
||||
@@ -292,7 +258,7 @@ async function createSession(env: QaGatewayCallEnv, label: string, key?: string)
|
||||
{
|
||||
timeoutMs: liveTurnTimeoutMs(env, 60_000),
|
||||
},
|
||||
);
|
||||
)) as { key?: string };
|
||||
const sessionKey = created.key?.trim();
|
||||
if (!sessionKey) {
|
||||
throw new Error("sessions.create returned no key");
|
||||
@@ -301,10 +267,7 @@ async function createSession(env: QaGatewayCallEnv, label: string, key?: string)
|
||||
}
|
||||
|
||||
async function readEffectiveTools(env: QaGatewayCallEnv, sessionKey: string) {
|
||||
const payload = await callGatewayWithSessionStoreLockRetry<{
|
||||
groups?: Array<{ tools?: Array<{ id?: string }> }>;
|
||||
}>(
|
||||
env,
|
||||
const payload = (await env.gateway.call(
|
||||
"tools.effective",
|
||||
{
|
||||
sessionKey,
|
||||
@@ -312,7 +275,7 @@ async function readEffectiveTools(env: QaGatewayCallEnv, sessionKey: string) {
|
||||
{
|
||||
timeoutMs: liveTurnTimeoutMs(env, 90_000),
|
||||
},
|
||||
);
|
||||
)) as { groups?: Array<{ tools?: Array<{ id?: string }> }> };
|
||||
const ids = new Set<string>();
|
||||
for (const group of payload.groups ?? []) {
|
||||
for (const tool of group.tools ?? []) {
|
||||
@@ -325,10 +288,7 @@ async function readEffectiveTools(env: QaGatewayCallEnv, sessionKey: string) {
|
||||
}
|
||||
|
||||
async function readSkillStatus(env: QaGatewayCallEnv, agentId = "qa") {
|
||||
const payload = await callGatewayWithSessionStoreLockRetry<{
|
||||
skills?: QaSkillStatusEntry[];
|
||||
}>(
|
||||
env,
|
||||
const payload = (await env.gateway.call(
|
||||
"skills.status",
|
||||
{
|
||||
agentId,
|
||||
@@ -336,7 +296,7 @@ async function readSkillStatus(env: QaGatewayCallEnv, agentId = "qa") {
|
||||
{
|
||||
timeoutMs: liveTurnTimeoutMs(env, 45_000),
|
||||
},
|
||||
);
|
||||
)) as { skills?: QaSkillStatusEntry[] };
|
||||
return payload.skills ?? [];
|
||||
}
|
||||
|
||||
|
||||
@@ -142,7 +142,8 @@ const defaultPublicDeprecatedExportsByEntrypointBudget = Object.freeze({
|
||||
"text-runtime": 192,
|
||||
"agent-runtime": 2,
|
||||
"channel-secret-runtime": 23,
|
||||
"agent-harness-runtime": 4,
|
||||
// +4: session-write lease no-op compatibility stubs through the 2026.10 train.
|
||||
"agent-harness-runtime": 8,
|
||||
"agent-config-primitives": 2,
|
||||
"command-auth": 78,
|
||||
discord: 47,
|
||||
@@ -318,7 +319,8 @@ export function readPluginSdkSurfaceBudgets(env: NodeJS.ProcessEnv = process.env
|
||||
// +1: shared ingress error factory projected through channel-message.
|
||||
// +1: shared ingress retention defaults projected through channel-message.
|
||||
// +1: shipped channel setup state-migration declaration during its migration window.
|
||||
1704,
|
||||
// +4: session-write lease no-op compatibility stubs through the 2026.10 train.
|
||||
1708,
|
||||
env,
|
||||
),
|
||||
publicWildcardReexports: readPluginSdkSurfaceBudgetEnv(
|
||||
|
||||
@@ -348,9 +348,6 @@ const providerAuthAliasMocks = vi.hoisted(() => ({
|
||||
},
|
||||
),
|
||||
}));
|
||||
const sessionWriteLockMocks = vi.hoisted(() => ({
|
||||
acquireSessionWriteLock: vi.fn(),
|
||||
}));
|
||||
vi.mock("../cli-runner.js", () => ({
|
||||
runCliAgent: runCliAgentMock,
|
||||
}));
|
||||
@@ -407,17 +404,6 @@ vi.mock("../embedded-agent.js", () => ({
|
||||
runEmbeddedAgent: runEmbeddedAgentMock,
|
||||
}));
|
||||
|
||||
vi.mock("../session-write-lock.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../session-write-lock.js")>(
|
||||
"../session-write-lock.js",
|
||||
);
|
||||
sessionWriteLockMocks.acquireSessionWriteLock.mockImplementation(actual.acquireSessionWriteLock);
|
||||
return {
|
||||
...actual,
|
||||
acquireSessionWriteLock: sessionWriteLockMocks.acquireSessionWriteLock,
|
||||
};
|
||||
});
|
||||
|
||||
function makeCliResult(text: string): EmbeddedAgentRunResult {
|
||||
return {
|
||||
payloads: [{ text }],
|
||||
@@ -665,7 +651,6 @@ describe("CLI attempt execution", () => {
|
||||
hasClaudeLiveSessionForOwnerMock.mockReturnValue(false);
|
||||
providerAuthAliasMocks.resolveProviderAuthAliasMap.mockClear();
|
||||
providerAuthAliasMocks.resolveProviderIdForAuth.mockClear();
|
||||
sessionWriteLockMocks.acquireSessionWriteLock.mockClear();
|
||||
cliBackendsTesting.setDepsForTest({
|
||||
resolvePluginSetupCliBackend: () => undefined,
|
||||
resolvePluginSetupRegistry: () => ({ cliBackends: [] }) as never,
|
||||
@@ -2208,36 +2193,6 @@ describe("CLI attempt execution", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("holds the session-key lease for the embedded assistant gap-fill", async () => {
|
||||
const sessionKey = "agent:main:direct:gap-fill-lease";
|
||||
const sessionEntry = makeSessionEntry("session-gap-fill-lease");
|
||||
await writeSessionStoreSeed({ [sessionKey]: sessionEntry });
|
||||
const release = vi.fn();
|
||||
sessionWriteLockMocks.acquireSessionWriteLock.mockResolvedValueOnce({ release });
|
||||
|
||||
await persistCliTurnTranscript({
|
||||
body: "ignored prompt",
|
||||
result: makeCliResult("runtime answer"),
|
||||
sessionId: sessionEntry.sessionId,
|
||||
sessionKey,
|
||||
sessionEntry,
|
||||
storePath,
|
||||
sessionAgentId: "main",
|
||||
sessionCwd: tmpDir,
|
||||
config: {},
|
||||
embeddedAssistantGapFill: true,
|
||||
});
|
||||
|
||||
expect(sessionWriteLockMocks.acquireSessionWriteLock).toHaveBeenCalledOnce();
|
||||
expect(sessionWriteLockMocks.acquireSessionWriteLock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
sessionFile: expect.any(String),
|
||||
targetKind: "session-key",
|
||||
}),
|
||||
);
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("persists a media-only ACP user turn when the reply is empty", async () => {
|
||||
const sessionKey = "agent:main:direct:acp-media-only";
|
||||
const sessionFile = path.join(tmpDir, "session-acp-media-only.jsonl");
|
||||
|
||||
@@ -22,7 +22,6 @@ import { messageToolOwnsVisibleReply } from "../../auto-reply/source-reply-deliv
|
||||
import type { ThinkLevel, VerboseLevel } from "../../auto-reply/thinking.js";
|
||||
import { resolveSessionAuthProfileOverrideSource } from "../../config/sessions/auth-profile-override-provenance.js";
|
||||
import { persistSessionTranscriptTurn } from "../../config/sessions/session-accessor.js";
|
||||
import { acquireOwnedSessionTranscriptWriteLock } from "../../config/sessions/transcript-write-context.js";
|
||||
import { readTailAssistantTextFromSessionTranscript } from "../../config/sessions/transcript.js";
|
||||
import type { SessionEntry } from "../../config/sessions/types.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
@@ -83,17 +82,12 @@ import { resolveCliRuntimeExecutionProvider } from "../model-runtime-aliases.js"
|
||||
import { isCliProvider } from "../model-selection.js";
|
||||
import { resolveOpenAIRuntimeProvider } from "../openai-routing.js";
|
||||
import { hasVerifiedRequesterCompletionHandoff } from "../requester-tool-policy.js";
|
||||
import { resolveAgentRunSessionTarget, type AgentRunSessionTarget } from "../run-session-target.js";
|
||||
import type { AgentRunSessionTarget } from "../run-session-target.js";
|
||||
import { resolveAgentRunAbortLifecycleFields } from "../run-termination.js";
|
||||
import { buildAgentRuntimeAuthPlan } from "../runtime-plan/auth.js";
|
||||
import type { AgentMessage } from "../runtime/index.js";
|
||||
import { resolveSandboxRuntimeStatus } from "../sandbox/runtime-status.js";
|
||||
import { withLocalSessionPlacementTurnAdmission } from "../session-placement-admission.js";
|
||||
import {
|
||||
acquireSessionWriteLock,
|
||||
resolveSessionWriteLockOptions,
|
||||
resolveSessionWriteLockTargetKey,
|
||||
} from "../session-write-lock.js";
|
||||
import { buildUsageWithNoCost } from "../stream-message-shared.js";
|
||||
import {
|
||||
isSubagentAnnounceCompletionHandoff,
|
||||
@@ -520,37 +514,7 @@ export async function persistCliTurnTranscript(params: {
|
||||
return await persist();
|
||||
}
|
||||
|
||||
const sessionTarget = await resolveAgentRunSessionTarget({
|
||||
agentId: params.sessionAgentId,
|
||||
config: params.config,
|
||||
missingSessionKey: "resolve-existing",
|
||||
sessionFile: params.sessionFile,
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionTarget: {
|
||||
agentId: params.sessionAgentId,
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
...(params.storePath ? { storePath: params.storePath } : {}),
|
||||
...(params.threadId !== undefined ? { threadId: params.threadId } : {}),
|
||||
},
|
||||
});
|
||||
const sessionLock =
|
||||
(await acquireOwnedSessionTranscriptWriteLock({
|
||||
sessionFile: params.sessionFile,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionTarget,
|
||||
})) ??
|
||||
(await acquireSessionWriteLock({
|
||||
sessionFile: resolveSessionWriteLockTargetKey(sessionTarget),
|
||||
targetKind: "session-key",
|
||||
...resolveSessionWriteLockOptions(params.config),
|
||||
}));
|
||||
try {
|
||||
return await persist();
|
||||
} finally {
|
||||
await sessionLock.release();
|
||||
}
|
||||
return await persist();
|
||||
}
|
||||
|
||||
export function runAgentAttempt(params: {
|
||||
|
||||
@@ -361,7 +361,7 @@ function stripErrorPrefix(raw: string): string {
|
||||
return raw.replace(/^error:\s*/i, "").trim();
|
||||
}
|
||||
|
||||
export function inferSignalStatus(signal: FailoverSignal): number | undefined {
|
||||
function inferSignalStatus(signal: FailoverSignal): number | undefined {
|
||||
if (typeof signal.status === "number" && Number.isFinite(signal.status)) {
|
||||
return signal.status;
|
||||
}
|
||||
|
||||
@@ -47,9 +47,6 @@ export const hookRunner = {
|
||||
runAfterCompaction: vi.fn(async () => undefined),
|
||||
};
|
||||
|
||||
export const acquireSessionWriteLockMock = vi.fn(async (_params?: unknown) => ({
|
||||
release: vi.fn(async () => {}),
|
||||
}));
|
||||
export const resolveContextEngineMock = vi.fn(async () => ({
|
||||
info: { ownsCompaction: true as boolean },
|
||||
compact: contextEngineCompactMock,
|
||||
@@ -566,7 +563,6 @@ export function resetCompactHooksHarnessMocks(): void {
|
||||
hookRunner.runAfterCompaction.mockResolvedValue(undefined);
|
||||
|
||||
acquireAgentRunPreparedModelRuntimeMock.mockClear();
|
||||
acquireSessionWriteLockMock.mockClear();
|
||||
|
||||
resolveContextEngineMock.mockReset();
|
||||
resolveContextEngineMock.mockResolvedValue({
|
||||
@@ -638,7 +634,6 @@ export async function loadCompactHooksHarness(): Promise<{
|
||||
compactEmbeddedAgentSessionDirect: typeof import("./compact.js").compactEmbeddedAgentSessionDirect;
|
||||
compactEmbeddedAgentSession: typeof import("./compact.queued.js").compactEmbeddedAgentSession;
|
||||
testing: typeof import("./compact.js").testing;
|
||||
withOwnedSessionTranscriptWrites: typeof import("../../config/sessions/transcript-write-context.js").withOwnedSessionTranscriptWrites;
|
||||
onSessionTranscriptUpdate: typeof import("../../sessions/transcript-events.js").onSessionTranscriptUpdate;
|
||||
onInternalSessionTranscriptUpdate: typeof import("../../sessions/transcript-events.js").onInternalSessionTranscriptUpdate;
|
||||
}> {
|
||||
@@ -795,23 +790,6 @@ export async function loadCompactHooksHarness(): Promise<{
|
||||
resolveSandboxContext: resolveSandboxContextMock,
|
||||
}));
|
||||
|
||||
vi.doMock("../session-write-lock.js", async () => {
|
||||
const { resolveSessionWriteLockTargetKey } = await vi.importActual<
|
||||
typeof import("../session-write-lock.js")
|
||||
>("../session-write-lock.js");
|
||||
return {
|
||||
acquireSessionWriteLock: acquireSessionWriteLockMock,
|
||||
resolveSessionLockMaxHoldFromTimeout: vi.fn(() => 0),
|
||||
resolveSessionWriteLockAcquireTimeoutMs: vi.fn(() => 60_000),
|
||||
resolveSessionWriteLockOptions: vi.fn(() => ({
|
||||
timeoutMs: 60_000,
|
||||
staleMs: 1_800_000,
|
||||
maxHoldMs: 300_000,
|
||||
})),
|
||||
resolveSessionWriteLockTargetKey,
|
||||
};
|
||||
});
|
||||
|
||||
vi.doMock("../../context-engine/init.js", () => ({
|
||||
ensureContextEnginesInitialized: vi.fn(),
|
||||
}));
|
||||
@@ -1099,20 +1077,17 @@ export async function loadCompactHooksHarness(): Promise<{
|
||||
};
|
||||
});
|
||||
|
||||
const [compactModule, compactQueuedModule, transcriptEvents, transcriptWriteContext] =
|
||||
await Promise.all([
|
||||
import("./compact.js"),
|
||||
import("./compact.queued.js"),
|
||||
import("../../sessions/transcript-events.js"),
|
||||
import("../../config/sessions/transcript-write-context.js"),
|
||||
]);
|
||||
const [compactModule, compactQueuedModule, transcriptEvents] = await Promise.all([
|
||||
import("./compact.js"),
|
||||
import("./compact.queued.js"),
|
||||
import("../../sessions/transcript-events.js"),
|
||||
]);
|
||||
|
||||
return {
|
||||
...compactModule,
|
||||
compactEmbeddedAgentSession: compactQueuedModule.compactEmbeddedAgentSession,
|
||||
onSessionTranscriptUpdate: transcriptEvents.onSessionTranscriptUpdate,
|
||||
onInternalSessionTranscriptUpdate: transcriptEvents.onInternalSessionTranscriptUpdate,
|
||||
withOwnedSessionTranscriptWrites: transcriptWriteContext.withOwnedSessionTranscriptWrites,
|
||||
};
|
||||
}
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
|
||||
@@ -11,7 +11,6 @@ import { createReplyOperation } from "../../auto-reply/reply/reply-run-registry.
|
||||
import { upsertSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js";
|
||||
import {
|
||||
acquireSessionWriteLockMock,
|
||||
acquireAgentRunPreparedModelRuntimeMock,
|
||||
applyExtraParamsToAgentMock,
|
||||
applyAgentCompactionSettingsFromConfigMock,
|
||||
@@ -72,7 +71,6 @@ let compactEmbeddedAgentSession: typeof import("./compact.queued.js").compactEmb
|
||||
let compactTesting: typeof import("./compact.js").testing;
|
||||
let onSessionTranscriptUpdate: typeof import("../../sessions/transcript-events.js").onSessionTranscriptUpdate;
|
||||
let onInternalSessionTranscriptUpdate: typeof import("../../sessions/transcript-events.js").onInternalSessionTranscriptUpdate;
|
||||
let withOwnedSessionTranscriptWrites: typeof import("../../config/sessions/transcript-write-context.js").withOwnedSessionTranscriptWrites;
|
||||
|
||||
const TEST_SESSION_ID = "session-1";
|
||||
const TEST_SESSION_KEY = "agent:main:session-1";
|
||||
@@ -308,7 +306,6 @@ beforeAll(async () => {
|
||||
compactTesting = loaded.testing;
|
||||
onSessionTranscriptUpdate = loaded.onSessionTranscriptUpdate;
|
||||
onInternalSessionTranscriptUpdate = loaded.onInternalSessionTranscriptUpdate;
|
||||
withOwnedSessionTranscriptWrites = loaded.withOwnedSessionTranscriptWrites;
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -332,41 +329,6 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => {
|
||||
resetCompactSessionStateMocks();
|
||||
});
|
||||
|
||||
it("acquires the normal session lock without process-wide reentry", async () => {
|
||||
const result = await compactEmbeddedAgentSessionDirect(wrappedCompactionArgs());
|
||||
|
||||
expect(result).toMatchObject({ ok: true, compacted: true });
|
||||
const lockOptions = mockCallArg(acquireSessionWriteLockMock);
|
||||
expect(lockOptions).toMatchObject({
|
||||
sessionFile: expect.any(String),
|
||||
targetKind: "session-key",
|
||||
});
|
||||
expect(lockOptions.sessionFile).not.toBe(TEST_SESSION_KEY);
|
||||
expect(lockOptions).not.toHaveProperty("allowReentrant");
|
||||
});
|
||||
|
||||
it("reuses the matching logical writer lock during direct compaction", async () => {
|
||||
const withSessionWriteLockCall = vi.fn();
|
||||
const withSessionWriteLock = async <T>(run: () => Promise<T> | T): Promise<T> => {
|
||||
withSessionWriteLockCall();
|
||||
return await run();
|
||||
};
|
||||
|
||||
const result = await withOwnedSessionTranscriptWrites(
|
||||
{
|
||||
sessionKey: TEST_SESSION_KEY,
|
||||
sessionTarget: wrappedCompactionArgs().sessionTarget,
|
||||
assertOwned: () => undefined,
|
||||
withSessionWriteLock,
|
||||
},
|
||||
async () => await compactEmbeddedAgentSessionDirect(wrappedCompactionArgs()),
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ ok: true, compacted: true });
|
||||
expect(withSessionWriteLockCall).toHaveBeenCalledOnce();
|
||||
expect(acquireSessionWriteLockMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails closed before generic compaction for a model-locked native session", async () => {
|
||||
const result = await compactEmbeddedAgentSessionDirect({
|
||||
sessionId: "session-1",
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { formatSqliteSessionFileMarker } from "../../config/sessions/legacy-sqlite-marker.js";
|
||||
import { acquireOwnedSessionTranscriptWriteLock } from "../../config/sessions/transcript-write-context.js";
|
||||
/**
|
||||
* Executes compaction while owning the transcript lock, session lifecycle,
|
||||
* hooks, checkpoint, and optional successor transcript rotation.
|
||||
@@ -24,12 +23,6 @@ import { pickFallbackThinkingLevel } from "../embedded-agent-helpers.js";
|
||||
import { resolveAgentRunSessionTarget } from "../run-session-target.js";
|
||||
import { guardSessionManager } from "../session-tool-result-guard-wrapper.js";
|
||||
import { sanitizeToolUseResultPairing } from "../session-transcript-repair.js";
|
||||
import {
|
||||
acquireSessionWriteLock,
|
||||
resolveSessionLockMaxHoldFromTimeout,
|
||||
resolveSessionWriteLockTargetKey,
|
||||
resolveSessionWriteLockOptions,
|
||||
} from "../session-write-lock.js";
|
||||
import { agentSessionAutomaticCompaction } from "../sessions/agent-session-compaction.js";
|
||||
import { createAgentSession, estimateTokens, SessionManager } from "../sessions/index.js";
|
||||
import { getModelRegistryRuntime } from "../sessions/model-registry-runtime.js";
|
||||
@@ -119,21 +112,6 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti
|
||||
sessionKey: params.sessionKey,
|
||||
sessionTarget: params.sessionTarget,
|
||||
});
|
||||
const sessionLock =
|
||||
(await acquireOwnedSessionTranscriptWriteLock({
|
||||
sessionFile: params.sessionFile,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionTarget,
|
||||
})) ??
|
||||
(await acquireSessionWriteLock({
|
||||
sessionFile: resolveSessionWriteLockTargetKey(sessionTarget),
|
||||
targetKind: "session-key",
|
||||
...resolveSessionWriteLockOptions(params.config, {
|
||||
maxHoldMsFallback: resolveSessionLockMaxHoldFromTimeout({
|
||||
timeoutMs: compactionTimeoutMs,
|
||||
}),
|
||||
}),
|
||||
}));
|
||||
try {
|
||||
const transcriptPolicy = runtimePlan.transcript.resolvePolicy(runtimePlanModelContext);
|
||||
const sessionManager = guardSessionManager(SessionManager.open(sessionTarget), {
|
||||
@@ -566,7 +544,6 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti
|
||||
}
|
||||
} finally {
|
||||
await runtime.disposeToolRuntimes();
|
||||
await sessionLock.release();
|
||||
}
|
||||
} catch (err) {
|
||||
const reason = resolveCompactionFailureReason({
|
||||
|
||||
@@ -4,10 +4,10 @@ import { expectDefined } from "@openclaw/normalization-core";
|
||||
import type { StreamFn } from "openclaw/plugin-sdk/agent-core";
|
||||
import type { Model } from "openclaw/plugin-sdk/llm";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionTranscriptWriterClaimReboundError } from "../../config/sessions/transcript-write-context.js";
|
||||
import { isSecretValueRegisteredForRedaction } from "../../logging/secret-redaction-registry.js";
|
||||
import { mintSecretSentinel, resolveSecretSentinel } from "../../secrets/sentinel.js";
|
||||
import { prepareGooglePromptCacheStreamFn } from "./google-prompt-cache.js";
|
||||
import { EmbeddedAttemptSessionTakeoverError } from "./run/attempt.session-lock.js";
|
||||
|
||||
type SessionCustomEntry = {
|
||||
type: "custom";
|
||||
@@ -488,9 +488,9 @@ describe("google prompt cache", () => {
|
||||
expect(getCapturedPayload()?.cachedContent).toBe("cachedContents/system-cache-2");
|
||||
});
|
||||
|
||||
it("propagates session takeover errors from cache entry persistence", async () => {
|
||||
it("propagates writer-claim rebound from cache entry persistence", async () => {
|
||||
const now = 2_500_000;
|
||||
const takeoverError = new EmbeddedAttemptSessionTakeoverError("/tmp/session.jsonl");
|
||||
const takeoverError = new SessionTranscriptWriterClaimReboundError("agent:main:test");
|
||||
const sessionManager = {
|
||||
appendCustomEntry: vi.fn(async () => {
|
||||
throw takeoverError;
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
resolveExpiresAtMsFromDurationMs,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { SessionTranscriptWriterClaimReboundError } from "../../config/sessions/transcript-write-context.js";
|
||||
import { parseGeminiAuth } from "../../infra/gemini-auth.js";
|
||||
import { normalizeGoogleApiBaseUrl } from "../../infra/google-api-base-url.js";
|
||||
import { cancelUnreadResponseBody, readResponseWithLimit } from "../../infra/http-body.js";
|
||||
@@ -28,10 +29,8 @@ import {
|
||||
import { resolveProviderRequestHeaders } from "../provider-request-config.js";
|
||||
import { buildGuardedModelFetch } from "../provider-transport-fetch.js";
|
||||
import type { StreamFn } from "../runtime/index.js";
|
||||
import { isSessionWriteLockAcquireError } from "../session-write-lock-error.js";
|
||||
import { log } from "./logger.js";
|
||||
import { isGooglePromptCacheEligible, resolveCacheRetention } from "./prompt-cache-retention.js";
|
||||
import { EmbeddedAttemptSessionTakeoverError } from "./run/attempt.session-lock.js";
|
||||
|
||||
const GOOGLE_PROMPT_CACHE_CUSTOM_TYPE = "openclaw.google-prompt-cache";
|
||||
// CachedContent metadata responses are tiny (name + expireTime); cap the read so
|
||||
@@ -196,7 +195,7 @@ async function appendGooglePromptCacheEntry(
|
||||
try {
|
||||
await sessionManager.appendCustomEntry(GOOGLE_PROMPT_CACHE_CUSTOM_TYPE, entry);
|
||||
} catch (err) {
|
||||
if (err instanceof EmbeddedAttemptSessionTakeoverError || isSessionWriteLockAcquireError(err)) {
|
||||
if (err instanceof SessionTranscriptWriterClaimReboundError) {
|
||||
throw err;
|
||||
}
|
||||
// ignore persistence failures
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Coverage for external cancellation, timeout, and session-lock release paths.
|
||||
// Coverage for external cancellation and timeout paths.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { EmbeddedAgentQueueHandle } from "../runs.js";
|
||||
import {
|
||||
@@ -8,8 +8,6 @@ import {
|
||||
} from "./attempt-abort.js";
|
||||
import { createEmbeddedAttemptSessionSettleTracker } from "./attempt-session-settle.js";
|
||||
import { prepareEmbeddedAttemptTimeout } from "./attempt-timeout-prepare.js";
|
||||
import { createEmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
import { SESSIONS_YIELD_ABORT_REASON } from "./attempt.sessions-yield.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
countActiveToolExecutions: vi.fn(() => 0),
|
||||
@@ -153,7 +151,6 @@ describe("createEmbeddedAttemptExternalAbortController", () => {
|
||||
const state = createAbortState();
|
||||
const session = createTrackedSessionAbort();
|
||||
const onAttemptTimeout = vi.fn();
|
||||
const releaseHeldLockForAbort = vi.fn(async () => {});
|
||||
const attempt = {
|
||||
abortSignal: source.signal,
|
||||
onAttemptTimeout,
|
||||
@@ -178,7 +175,6 @@ describe("createEmbeddedAttemptExternalAbortController", () => {
|
||||
isProbeSession: true,
|
||||
log: { warn: vi.fn() },
|
||||
runAbortController,
|
||||
sessionLockController: { releaseHeldLockForAbort },
|
||||
state: state.port,
|
||||
});
|
||||
controller.setRunAbort(abortRun);
|
||||
@@ -210,7 +206,6 @@ describe("createEmbeddedAttemptExternalAbortController", () => {
|
||||
expect(onAttemptTimeout).toHaveBeenCalledOnce();
|
||||
expect(session.abort).toHaveBeenCalledExactlyOnceWith(reason);
|
||||
expect(mocks.markActiveEmbeddedRunAbandoned).toHaveBeenCalledOnce();
|
||||
expect(releaseHeldLockForAbort).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
timeout.clearTimers();
|
||||
controller.dispose();
|
||||
@@ -241,42 +236,13 @@ describe("createEmbeddedAttemptExternalAbortController", () => {
|
||||
});
|
||||
|
||||
describe("createEmbeddedAttemptRunAbort", () => {
|
||||
it("releases session ownership non-terminally for sessions_yield handoff", async () => {
|
||||
const releaseHeldLockForAbort = vi.fn(async () => {});
|
||||
const abortRun = createEmbeddedAttemptRunAbort({
|
||||
abortActiveSession: vi.fn(async () => {}),
|
||||
activeSession: { abortCompaction: vi.fn(), isCompacting: false },
|
||||
attempt: {
|
||||
runId: "run-yield",
|
||||
sessionFile: "agent:main:main",
|
||||
sessionId: "session-yield",
|
||||
sessionKey: "agent:main:main",
|
||||
},
|
||||
getQueueHandle: () => undefined,
|
||||
isProbeSession: false,
|
||||
log: { warn: vi.fn() },
|
||||
runAbortController: new AbortController(),
|
||||
sessionLockController: { releaseHeldLockForAbort },
|
||||
state: createAbortState().port,
|
||||
});
|
||||
|
||||
abortRun(false, SESSIONS_YIELD_ABORT_REASON);
|
||||
await vi.waitFor(() => {
|
||||
expect(releaseHeldLockForAbort).toHaveBeenCalledWith({
|
||||
reason: SESSIONS_YIELD_ABORT_REASON,
|
||||
terminal: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("settles timeout state, session work, queue ownership, and the lock", async () => {
|
||||
it("settles timeout state, session work, and queue ownership", async () => {
|
||||
const state = createAbortState();
|
||||
const timeoutReason = new Error("attempt deadline");
|
||||
timeoutReason.name = "TimeoutError";
|
||||
const abortCompaction = vi.fn();
|
||||
const abortActiveSession = vi.fn(async () => {});
|
||||
const onAttemptTimeout = vi.fn();
|
||||
const releaseHeldLockForAbort = vi.fn(async () => {});
|
||||
const queueHandle = {} as EmbeddedAgentQueueHandle;
|
||||
const runAbortController = new AbortController();
|
||||
mocks.countActiveToolExecutions.mockReturnValue(1);
|
||||
@@ -294,7 +260,6 @@ describe("createEmbeddedAttemptRunAbort", () => {
|
||||
isProbeSession: false,
|
||||
log: { warn: vi.fn() },
|
||||
runAbortController,
|
||||
sessionLockController: { releaseHeldLockForAbort },
|
||||
state: state.port,
|
||||
});
|
||||
|
||||
@@ -315,60 +280,15 @@ describe("createEmbeddedAttemptRunAbort", () => {
|
||||
sessionFile: "/tmp/session.jsonl",
|
||||
reason: "timeout",
|
||||
});
|
||||
expect(releaseHeldLockForAbort).toHaveBeenCalledTimes(1);
|
||||
expect(releaseHeldLockForAbort).toHaveBeenCalledWith({
|
||||
reason: timeoutReason,
|
||||
terminal: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a timeout reason through the abort path to a late prompt handoff", async () => {
|
||||
const sessionLockController = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
const timeoutReason = new Error("cron setup timed out");
|
||||
timeoutReason.name = "TimeoutError";
|
||||
const abortRun = createEmbeddedAttemptRunAbort({
|
||||
abortActiveSession: vi.fn(async () => {}),
|
||||
activeSession: { abortCompaction: vi.fn(), isCompacting: false },
|
||||
attempt: {
|
||||
onAttemptTimeout: vi.fn(),
|
||||
runId: "run-timeout-handoff",
|
||||
sessionFile: "agent:main:main",
|
||||
sessionId: "session-timeout-handoff",
|
||||
sessionKey: "agent:main:main",
|
||||
},
|
||||
getQueueHandle: () => undefined,
|
||||
isProbeSession: false,
|
||||
log: { warn: vi.fn() },
|
||||
runAbortController: new AbortController(),
|
||||
sessionLockController,
|
||||
state: createAbortState().port,
|
||||
});
|
||||
|
||||
abortRun(true, timeoutReason);
|
||||
|
||||
await expect(sessionLockController.releaseForPrompt()).rejects.toBe(timeoutReason);
|
||||
await sessionLockController.dispose();
|
||||
});
|
||||
|
||||
it("logs lock release failures without replacing the manual abort reason", async () => {
|
||||
// Abort cleanup must not replace the original timeout/manual-abort reason
|
||||
// with a secondary lock-release failure.
|
||||
const state = createAbortState();
|
||||
it("preserves a manual abort reason", () => {
|
||||
const abortReason = new Error("manual abort");
|
||||
const releaseError = new Error("locked");
|
||||
const releaseHeldLockForAbort = vi.fn(async () => {
|
||||
throw releaseError;
|
||||
});
|
||||
const warn = vi.fn();
|
||||
const runAbortController = new AbortController();
|
||||
const abortRun = createEmbeddedAttemptRunAbort({
|
||||
abortActiveSession: vi.fn(async () => {}),
|
||||
activeSession: { abortCompaction: vi.fn(), isCompacting: false },
|
||||
attempt: {
|
||||
onAttemptTimeout: vi.fn(),
|
||||
runId: "run-manual",
|
||||
sessionFile: "/tmp/session.jsonl",
|
||||
sessionId: "session-manual",
|
||||
@@ -376,21 +296,13 @@ describe("createEmbeddedAttemptRunAbort", () => {
|
||||
},
|
||||
getQueueHandle: () => undefined,
|
||||
isProbeSession: false,
|
||||
log: { warn },
|
||||
log: { warn: vi.fn() },
|
||||
runAbortController,
|
||||
sessionLockController: { releaseHeldLockForAbort },
|
||||
state: state.port,
|
||||
state: createAbortState().port,
|
||||
});
|
||||
|
||||
abortRun(false, abortReason);
|
||||
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(runAbortController.signal.reason).toBe(abortReason);
|
||||
expect(releaseHeldLockForAbort).toHaveBeenCalledTimes(1);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
"failed to release session lock on abort: runId=run-manual Error: locked",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,12 +5,10 @@ import { countActiveToolExecutions } from "../../embedded-agent-subscribe.handle
|
||||
import { isSignalTimeoutReason } from "../../failover-error.js";
|
||||
import type { AgentSession } from "../../sessions/index.js";
|
||||
import { markActiveEmbeddedRunAbandoned, type EmbeddedAgentQueueHandle } from "../runs.js";
|
||||
import type { EmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
import { isSessionsYieldAbortError, isSessionsYieldAbortReason } from "./attempt.sessions-yield.js";
|
||||
import { shouldFlagCompactionTimeout } from "./compaction-timeout.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type AbortLockReleaseLog = {
|
||||
type AbortLog = {
|
||||
warn(message: string): void;
|
||||
};
|
||||
|
||||
@@ -162,9 +160,8 @@ export function createEmbeddedAttemptRunAbort(input: {
|
||||
>;
|
||||
getQueueHandle: () => EmbeddedAgentQueueHandle | undefined;
|
||||
isProbeSession: boolean;
|
||||
log: AbortLockReleaseLog;
|
||||
log: AbortLog;
|
||||
runAbortController: AbortController;
|
||||
sessionLockController: Pick<EmbeddedAttemptSessionLockController, "releaseHeldLockForAbort">;
|
||||
state: Pick<
|
||||
EmbeddedAttemptAbortStatePort,
|
||||
| "markAborted"
|
||||
@@ -197,7 +194,6 @@ export function createEmbeddedAttemptRunAbort(input: {
|
||||
}
|
||||
abortAccepted = true;
|
||||
input.state.markAborted();
|
||||
let effectiveReason = reason;
|
||||
if (isTimeout) {
|
||||
input.state.markTimedOut();
|
||||
if (
|
||||
@@ -207,7 +203,6 @@ export function createEmbeddedAttemptRunAbort(input: {
|
||||
input.state.markTimedOutDuringToolExecution();
|
||||
}
|
||||
const timeoutReason = reason instanceof Error ? reason : createTimeoutAbortReason();
|
||||
effectiveReason = timeoutReason;
|
||||
input.attempt.onAttemptTimeout?.(timeoutReason);
|
||||
input.runAbortController.abort(timeoutReason);
|
||||
} else {
|
||||
@@ -225,36 +220,5 @@ export function createEmbeddedAttemptRunAbort(input: {
|
||||
reason: "timeout",
|
||||
});
|
||||
}
|
||||
releaseEmbeddedAttemptSessionLockForAbort({
|
||||
sessionLockController: input.sessionLockController,
|
||||
log: input.log,
|
||||
runId: input.attempt.runId,
|
||||
abortKind: isTimeout ? "timeout abort" : "abort",
|
||||
reason: effectiveReason,
|
||||
terminal:
|
||||
isTimeout || (!isSessionsYieldAbortError(reason) && !isSessionsYieldAbortReason(reason)),
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the held session lock after an abort without blocking abort
|
||||
* propagation. Release failures are logged because the caller is already
|
||||
* unwinding the run and cannot safely await lock cleanup there.
|
||||
*/
|
||||
function releaseEmbeddedAttemptSessionLockForAbort(params: {
|
||||
sessionLockController: Pick<EmbeddedAttemptSessionLockController, "releaseHeldLockForAbort">;
|
||||
log: AbortLockReleaseLog;
|
||||
runId: string;
|
||||
abortKind: "abort" | "timeout abort";
|
||||
reason?: unknown;
|
||||
terminal: boolean;
|
||||
}): void {
|
||||
void params.sessionLockController
|
||||
.releaseHeldLockForAbort({ reason: params.reason, terminal: params.terminal })
|
||||
.catch((err: unknown) => {
|
||||
params.log.warn(
|
||||
`failed to release session lock on ${params.abortKind}: runId=${params.runId} ${String(err)}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -23,26 +23,21 @@ import {
|
||||
type buildContextEnginePromptCacheInfo,
|
||||
} from "./attempt.context-engine-helpers.js";
|
||||
import { buildAfterTurnRuntimeContextFromUsage } from "./attempt.prompt-helpers.js";
|
||||
import type { createEmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
import { shouldPersistCompletedBootstrapTurn } from "./attempt.thread-helpers.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type CacheTrace = ReturnType<typeof createCacheTrace>;
|
||||
type AnthropicPayloadLogger = ReturnType<typeof createAnthropicPayloadLogger>;
|
||||
type HookRunner = ReturnType<typeof getGlobalHookRunner>;
|
||||
type AttemptSessionLockController = Awaited<
|
||||
ReturnType<typeof createEmbeddedAttemptSessionLockController>
|
||||
>;
|
||||
type PromptCacheInfo = ReturnType<typeof buildContextEnginePromptCacheInfo>;
|
||||
type WithOwnedSessionWriteLock = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
type WithOwnedTranscriptWrite = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
|
||||
type CompleteEmbeddedAttemptAfterTurnInput = {
|
||||
attempt: EmbeddedRunAttemptParams;
|
||||
activeContextEngine?: ContextEngine;
|
||||
activeSession: AgentSession;
|
||||
sessionManager: SessionManager;
|
||||
sessionLockController: AttemptSessionLockController;
|
||||
withOwnedSessionWriteLock: WithOwnedSessionWriteLock;
|
||||
withOwnedTranscriptWrite: WithOwnedTranscriptWrite;
|
||||
state: {
|
||||
promptError: unknown;
|
||||
yieldAborted: boolean;
|
||||
@@ -102,7 +97,7 @@ export async function completeEmbeddedAttemptAfterTurn(
|
||||
messagesSnapshot: AgentMessage[];
|
||||
prePromptMessageCount: number;
|
||||
sessionManager?: SessionManager;
|
||||
withSessionManagerRewriteLock: WithOwnedSessionWriteLock;
|
||||
withSessionManagerRewriteLock: WithOwnedTranscriptWrite;
|
||||
}) => {
|
||||
await finalizeAttemptContextEngineTurn({
|
||||
contextEngine: activeContextEngine,
|
||||
@@ -179,13 +174,13 @@ export async function completeEmbeddedAttemptAfterTurn(
|
||||
prePromptMessageCount:
|
||||
state.contextEngineAfterTurnCheckpoint ?? state.prePromptMessageCount,
|
||||
sessionManager,
|
||||
withSessionManagerRewriteLock: input.withOwnedSessionWriteLock,
|
||||
withSessionManagerRewriteLock: input.withOwnedTranscriptWrite,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!state.beforeAgentFinalizeRevisionReason) {
|
||||
await input.withOwnedSessionWriteLock(async () => {
|
||||
await input.withOwnedTranscriptWrite(async () => {
|
||||
const lifecycleState = input.readLifecycleState();
|
||||
if (
|
||||
shouldPersistCompletedBootstrapTurn({
|
||||
|
||||
@@ -40,7 +40,7 @@ function createInput(hooks: PluginHookRegistration[] = []) {
|
||||
agent: { state },
|
||||
};
|
||||
const lock = { acquisitions: 0 };
|
||||
const withOwnedSessionWriteLock = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
const withOwnedTranscriptWrite = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
lock.acquisitions += 1;
|
||||
return await operation();
|
||||
};
|
||||
@@ -59,7 +59,7 @@ function createInput(hooks: PluginHookRegistration[] = []) {
|
||||
modelPrompt: "model prompt",
|
||||
sessionManager,
|
||||
systemPrompt: "system prompt",
|
||||
withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite,
|
||||
},
|
||||
lock,
|
||||
sessionManager,
|
||||
|
||||
@@ -14,7 +14,7 @@ type HookRunner = NonNullable<ReturnType<typeof getGlobalHookRunner>>;
|
||||
type BeforeAgentRunHookRunner = Pick<HookRunner, "hasHooks" | "runBeforeAgentRun">;
|
||||
type HookContext = Parameters<HookRunner["runBeforeAgentRun"]>[1];
|
||||
type AttemptSessionManager = Parameters<typeof flushSessionManagerTranscript>[0];
|
||||
type WithOwnedSessionWriteLock = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
type WithOwnedTranscriptWrite = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
|
||||
type BeforeAgentRunSession = {
|
||||
messages: AgentMessage[];
|
||||
@@ -38,7 +38,7 @@ export async function runEmbeddedAttemptBeforeAgentRun(input: {
|
||||
modelPrompt: string;
|
||||
sessionManager: AttemptSessionManager;
|
||||
systemPrompt: string;
|
||||
withOwnedSessionWriteLock: WithOwnedSessionWriteLock;
|
||||
withOwnedTranscriptWrite: WithOwnedTranscriptWrite;
|
||||
}): Promise<BeforeAgentRunBlockOutcome | undefined> {
|
||||
if (!input.hookRunner?.hasHooks("before_agent_run")) {
|
||||
return undefined;
|
||||
@@ -66,7 +66,7 @@ export async function runEmbeddedAttemptBeforeAgentRun(input: {
|
||||
},
|
||||
};
|
||||
try {
|
||||
await input.withOwnedSessionWriteLock(() => {
|
||||
await input.withOwnedTranscriptWrite(() => {
|
||||
input.sessionManager.appendMessage(
|
||||
redactedUserMessage as Parameters<typeof input.sessionManager.appendMessage>[0],
|
||||
);
|
||||
|
||||
@@ -89,8 +89,7 @@ function createFixture() {
|
||||
sessionLock: {
|
||||
compactionTimeoutMs: 1_000,
|
||||
ownedTranscriptWriteContext: {},
|
||||
sessionLockController: {},
|
||||
withOwnedSessionWriteLock: vi.fn(),
|
||||
withOwnedTranscriptWrite: vi.fn(),
|
||||
},
|
||||
setup: {
|
||||
effectiveFsWorkspaceOnly: false,
|
||||
|
||||
@@ -53,7 +53,6 @@ export async function runEmbeddedAttemptExecutionPhase(
|
||||
attempt,
|
||||
activeSession,
|
||||
sessionManager,
|
||||
sessionLockController: input.sessionLock.sessionLockController,
|
||||
ownedTranscriptWriteContext: input.sessionLock.ownedTranscriptWriteContext,
|
||||
runAbortController: input.runAbortController,
|
||||
externalAbortController: input.externalAbortController,
|
||||
|
||||
@@ -162,8 +162,7 @@ function createFixture() {
|
||||
},
|
||||
},
|
||||
sessionLock: {
|
||||
sessionLockController: {},
|
||||
withOwnedSessionWriteLock: vi.fn(),
|
||||
withOwnedTranscriptWrite: vi.fn(),
|
||||
},
|
||||
setup: {
|
||||
effectiveFsWorkspaceOnly: false,
|
||||
|
||||
@@ -172,8 +172,7 @@ export async function runEmbeddedAttemptSettledPhase(
|
||||
attempt,
|
||||
activeSession,
|
||||
sessionManager,
|
||||
sessionLockController: input.sessionLock.sessionLockController,
|
||||
withOwnedSessionWriteLock: input.sessionLock.withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite: input.sessionLock.withOwnedTranscriptWrite,
|
||||
getCompactionReserveTokens: () => settingsManager.getCompactionReserveTokens(),
|
||||
...(emptyExplicitToolAllowlistError ? { emptyExplicitToolAllowlistError } : {}),
|
||||
assembly: {
|
||||
@@ -308,8 +307,7 @@ export async function runEmbeddedAttemptSettledPhase(
|
||||
attempt,
|
||||
activeSession,
|
||||
sessionManager,
|
||||
sessionLockController: input.sessionLock.sessionLockController,
|
||||
withOwnedSessionWriteLock: input.sessionLock.withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite: input.sessionLock.withOwnedTranscriptWrite,
|
||||
waitForPendingEvents,
|
||||
repairedRejectedThinkingReplay: input.getRepairedRejectedThinkingReplay(),
|
||||
getRunAbortDeadlineAtMs,
|
||||
|
||||
@@ -6,18 +6,18 @@ import type {
|
||||
} from "./attempt-abort.js";
|
||||
import type { prepareEmbeddedAttemptBootstrap } from "./attempt-bootstrap-prepare.js";
|
||||
import type { prepareEmbeddedAttemptBundleTools } from "./attempt-bundle-tools.js";
|
||||
import type { prepareEmbeddedAttemptSessionLock } from "./attempt-session-lock-prepare.js";
|
||||
import type { prepareEmbeddedAttemptSessionRuntime } from "./attempt-session-runtime-prepare.js";
|
||||
import type { prepareEmbeddedAttemptSetup } from "./attempt-setup.js";
|
||||
import type { prepareEmbeddedAttemptStreamRuntime } from "./attempt-stream-runtime-prepare.js";
|
||||
import type { prepareEmbeddedAttemptSystemPrompt } from "./attempt-system-prompt-prepare.js";
|
||||
import type { prepareEmbeddedAttemptToolBase } from "./attempt-tool-base-prepare.js";
|
||||
import type { prepareEmbeddedAttemptToolCatalog } from "./attempt-tool-catalog.js";
|
||||
import type { prepareEmbeddedAttemptTranscriptLifecycle } from "./attempt-transcript-lifecycle-prepare.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type Prepared<T extends (...args: never[]) => unknown> = Awaited<ReturnType<T>>;
|
||||
type PreparedSetup = Prepared<typeof prepareEmbeddedAttemptSetup>;
|
||||
type PreparedSessionLock = Prepared<typeof prepareEmbeddedAttemptSessionLock>;
|
||||
type PreparedTranscriptLifecycle = Prepared<typeof prepareEmbeddedAttemptTranscriptLifecycle>;
|
||||
type StreamRuntimeInput = Parameters<typeof prepareEmbeddedAttemptStreamRuntime>[0];
|
||||
type AttemptContextEngine = NonNullable<StreamRuntimeInput["history"]["activeContextEngine"]>;
|
||||
|
||||
@@ -49,11 +49,8 @@ export type EmbeddedAttemptExecutionPhaseInput = {
|
||||
toolCatalog: ReturnType<typeof prepareEmbeddedAttemptToolCatalog>;
|
||||
};
|
||||
sessionLock: Pick<
|
||||
PreparedSessionLock,
|
||||
| "compactionTimeoutMs"
|
||||
| "ownedTranscriptWriteContext"
|
||||
| "sessionLockController"
|
||||
| "withOwnedSessionWriteLock"
|
||||
PreparedTranscriptLifecycle,
|
||||
"compactionTimeoutMs" | "ownedTranscriptWriteContext" | "withOwnedTranscriptWrite"
|
||||
>;
|
||||
setup: Pick<
|
||||
PreparedSetup,
|
||||
|
||||
@@ -57,8 +57,7 @@ describe("embedded attempt phase lifecycle state", () => {
|
||||
} as never,
|
||||
activeSession: activeSession as never,
|
||||
sessionManager: sessionManager as never,
|
||||
sessionLockController: {} as never,
|
||||
withOwnedSessionWriteLock: async (operation) => await operation(),
|
||||
withOwnedTranscriptWrite: async (operation) => await operation(),
|
||||
subscription: {
|
||||
toolMetas: [],
|
||||
waitForCompactionRetry: async () => {
|
||||
@@ -138,8 +137,7 @@ describe("embedded attempt phase lifecycle state", () => {
|
||||
} as never,
|
||||
activeSession: activeSession as never,
|
||||
sessionManager: sessionManager as never,
|
||||
sessionLockController: {} as never,
|
||||
withOwnedSessionWriteLock: async (operation) => await operation(),
|
||||
withOwnedTranscriptWrite: async (operation) => await operation(),
|
||||
subscription: {
|
||||
toolMetas: [{ toolName: "exec", asyncStarted: true }],
|
||||
waitForCompactionRetry: async () => {},
|
||||
@@ -224,8 +222,7 @@ describe("embedded attempt phase lifecycle state", () => {
|
||||
} as never,
|
||||
activeSession: activeSession as never,
|
||||
sessionManager: sessionManager as never,
|
||||
sessionLockController: {} as never,
|
||||
withOwnedSessionWriteLock: async (operation) => await operation(),
|
||||
withOwnedTranscriptWrite: async (operation) => await operation(),
|
||||
subscription: {
|
||||
toolMetas: [
|
||||
{ toolName: "read", isError: true },
|
||||
@@ -321,8 +318,7 @@ describe("embedded attempt phase lifecycle state", () => {
|
||||
} as never,
|
||||
activeSession: {} as never,
|
||||
sessionManager: { appendCustomEntry: vi.fn(), getLeafId: vi.fn(() => "terminal") } as never,
|
||||
sessionLockController: {} as never,
|
||||
withOwnedSessionWriteLock: async (operation) => await operation(),
|
||||
withOwnedTranscriptWrite: async (operation) => await operation(),
|
||||
state: {
|
||||
promptError: null,
|
||||
yieldAborted: false,
|
||||
@@ -371,8 +367,7 @@ describe("embedded attempt phase lifecycle state", () => {
|
||||
} as never,
|
||||
activeSession: {} as never,
|
||||
sessionManager: { appendCustomEntry: vi.fn() } as never,
|
||||
sessionLockController: {} as never,
|
||||
withOwnedSessionWriteLock: async (operation) => await operation(),
|
||||
withOwnedTranscriptWrite: async (operation) => await operation(),
|
||||
state: {
|
||||
promptError: abortError,
|
||||
yieldAborted: false,
|
||||
@@ -420,8 +415,7 @@ describe("embedded attempt phase lifecycle state", () => {
|
||||
} as never,
|
||||
activeSession: {} as never,
|
||||
sessionManager: { appendCustomEntry: vi.fn() } as never,
|
||||
sessionLockController: {} as never,
|
||||
withOwnedSessionWriteLock: async (operation) => {
|
||||
withOwnedTranscriptWrite: async (operation) => {
|
||||
aborted = true;
|
||||
return await operation();
|
||||
},
|
||||
@@ -473,8 +467,7 @@ describe("embedded attempt phase lifecycle state", () => {
|
||||
} as never,
|
||||
activeSession: {} as never,
|
||||
sessionManager: { appendCustomEntry: vi.fn() } as never,
|
||||
sessionLockController: {} as never,
|
||||
withOwnedSessionWriteLock: async (operation) => await operation(),
|
||||
withOwnedTranscriptWrite: async (operation) => await operation(),
|
||||
state: {
|
||||
promptError: null,
|
||||
yieldAborted: false,
|
||||
|
||||
@@ -16,7 +16,7 @@ type PromptDispatchState = PromptPreflightInput["state"];
|
||||
export async function dispatchEmbeddedAttemptPrompt(input: {
|
||||
attempt: EmbeddedRunAttemptParams;
|
||||
activeContextEngine?: PromptPreflightInput["activeContextEngine"];
|
||||
activeSession: PromptExecutionInput["session"] & PromptSubmissionInput["activeSession"];
|
||||
activeSession: PromptSubmissionInput["activeSession"];
|
||||
promptContext: PromptContext;
|
||||
getCompactionReserveTokens: () => number;
|
||||
publishState: (state: PromptDispatchState) => void;
|
||||
@@ -71,7 +71,6 @@ export async function dispatchEmbeddedAttemptPrompt(input: {
|
||||
...input.execution,
|
||||
attempt,
|
||||
prompt: promptContext.promptSubmission.prompt,
|
||||
session: activeSession,
|
||||
skipPromptSubmission: input.state.skipPromptSubmission,
|
||||
});
|
||||
|
||||
|
||||
@@ -6,11 +6,10 @@ const hoisted = vi.hoisted(() => ({
|
||||
isSessionsYieldAbortError: vi.fn(() => false),
|
||||
markYieldAborted: vi.fn(),
|
||||
persistSessionsYieldContextMessage: vi.fn(async () => undefined),
|
||||
releaseHeldLockForAbort: vi.fn(async () => undefined),
|
||||
releaseLeasedSteering: vi.fn(),
|
||||
stripSessionsYieldArtifacts: vi.fn(),
|
||||
waitForSessionsYieldAbortSettle: vi.fn(async () => undefined),
|
||||
withOwnedSessionWriteLock: vi.fn(async (operation: () => unknown) => await operation()),
|
||||
withOwnedTranscriptWrite: vi.fn(async (operation: () => unknown) => await operation()),
|
||||
}));
|
||||
|
||||
vi.mock("./attempt.sessions-yield.js", () => ({
|
||||
@@ -35,10 +34,7 @@ function createInput(overrides: Partial<PromptErrorInput> = {}): PromptErrorInpu
|
||||
handleMidTurnPrecheckRequest: hoisted.handleMidTurnPrecheckRequest,
|
||||
markYieldAborted: hoisted.markYieldAborted,
|
||||
releaseLeasedSteering: hoisted.releaseLeasedSteering,
|
||||
sessionLockController: {
|
||||
releaseHeldLockForAbort: hoisted.releaseHeldLockForAbort,
|
||||
},
|
||||
withOwnedSessionWriteLock: hoisted.withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite: hoisted.withOwnedTranscriptWrite,
|
||||
yieldAbortSettled: null,
|
||||
yieldDetected: false,
|
||||
yieldMessage: null,
|
||||
@@ -63,7 +59,7 @@ describe("handleEmbeddedAttemptPromptError", () => {
|
||||
expect(hoisted.releaseLeasedSteering).toHaveBeenCalledWith(error);
|
||||
});
|
||||
|
||||
it("routes mid-turn prechecks under the owned session lock", async () => {
|
||||
it("routes mid-turn prechecks under the owned transcript context", async () => {
|
||||
const request = {
|
||||
route: "compact_only",
|
||||
estimatedPromptTokens: 12,
|
||||
@@ -77,7 +73,7 @@ describe("handleEmbeddedAttemptPromptError", () => {
|
||||
|
||||
await expect(handleEmbeddedAttemptPromptError(createInput({ error }))).resolves.toEqual({});
|
||||
|
||||
expect(hoisted.withOwnedSessionWriteLock).toHaveBeenCalledOnce();
|
||||
expect(hoisted.withOwnedTranscriptWrite).toHaveBeenCalledOnce();
|
||||
expect(hoisted.handleMidTurnPrecheckRequest).toHaveBeenCalledWith(request);
|
||||
});
|
||||
|
||||
@@ -100,7 +96,6 @@ describe("handleEmbeddedAttemptPromptError", () => {
|
||||
runId: "run-1",
|
||||
sessionId: "session-1",
|
||||
});
|
||||
expect(hoisted.releaseHeldLockForAbort).toHaveBeenCalledOnce();
|
||||
expect(hoisted.stripSessionsYieldArtifacts).toHaveBeenCalledWith(input.activeSession);
|
||||
expect(hoisted.persistSessionsYieldContextMessage).toHaveBeenCalledWith(
|
||||
input.activeSession,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/** Classifies prompt failures and performs yield or mid-turn recovery. */
|
||||
import type { AgentSession } from "../../sessions/index.js";
|
||||
import type { EmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
import {
|
||||
isSessionsYieldAbortError,
|
||||
persistSessionsYieldContextMessage,
|
||||
@@ -11,11 +10,7 @@ import { isMidTurnPrecheckSignal, type MidTurnPrecheckRequest } from "./midturn-
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type PromptErrorAttempt = Pick<EmbeddedRunAttemptParams, "runId" | "sessionId">;
|
||||
type PromptErrorSessionLockController = Pick<
|
||||
EmbeddedAttemptSessionLockController,
|
||||
"releaseHeldLockForAbort"
|
||||
>;
|
||||
type WithOwnedSessionWriteLock = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
type WithOwnedTranscriptWrite = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
|
||||
type EmbeddedAttemptPromptErrorOutcome = {
|
||||
promptFailure?: {
|
||||
@@ -31,8 +26,7 @@ export async function handleEmbeddedAttemptPromptError(input: {
|
||||
handleMidTurnPrecheckRequest: (request: MidTurnPrecheckRequest) => void;
|
||||
markYieldAborted: () => void;
|
||||
releaseLeasedSteering: (error?: unknown) => void;
|
||||
sessionLockController: PromptErrorSessionLockController;
|
||||
withOwnedSessionWriteLock: WithOwnedSessionWriteLock;
|
||||
withOwnedTranscriptWrite: WithOwnedTranscriptWrite;
|
||||
yieldAbortSettled: Promise<void> | null;
|
||||
yieldDetected: boolean;
|
||||
yieldMessage: string | null;
|
||||
@@ -47,8 +41,7 @@ export async function handleEmbeddedAttemptPromptError(input: {
|
||||
runId: input.attempt.runId,
|
||||
sessionId: input.attempt.sessionId,
|
||||
});
|
||||
await input.sessionLockController.releaseHeldLockForAbort({ terminal: false });
|
||||
await input.withOwnedSessionWriteLock(async () => {
|
||||
await input.withOwnedTranscriptWrite(async () => {
|
||||
stripSessionsYieldArtifacts(input.activeSession);
|
||||
if (input.yieldMessage) {
|
||||
await persistSessionsYieldContextMessage(input.activeSession, input.yieldMessage);
|
||||
@@ -59,7 +52,7 @@ export async function handleEmbeddedAttemptPromptError(input: {
|
||||
|
||||
if (isMidTurnPrecheckSignal(input.error)) {
|
||||
const request = input.error.request;
|
||||
await input.withOwnedSessionWriteLock(() => {
|
||||
await input.withOwnedTranscriptWrite(() => {
|
||||
input.handleMidTurnPrecheckRequest(request);
|
||||
});
|
||||
return {};
|
||||
|
||||
@@ -1,14 +1,8 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const hoisted = vi.hoisted(() => ({
|
||||
canAdvanceSessionEntryCache: vi.fn(() => true),
|
||||
detectAndLoadPromptImages: vi.fn(),
|
||||
installPromptSubmissionLockRelease: vi.fn((_input: Record<string, unknown>) => undefined),
|
||||
publishOwnedSessionFileSnapshot: vi.fn(() => true),
|
||||
reacquireAfterPrompt: vi.fn(async () => undefined),
|
||||
releaseForPrompt: vi.fn(async () => undefined),
|
||||
resolveImageSanitizationLimits: vi.fn(() => ({ maxDimensionPx: 2048 })),
|
||||
withSessionWriteLock: vi.fn(async (operation: () => unknown) => await operation()),
|
||||
}));
|
||||
|
||||
vi.mock("@openclaw/media-core/constants", () => ({
|
||||
@@ -22,9 +16,6 @@ vi.mock("../../image-sanitization.js", () => ({
|
||||
vi.mock("./images.js", () => ({
|
||||
detectAndLoadPromptImages: hoisted.detectAndLoadPromptImages,
|
||||
}));
|
||||
vi.mock("./attempt.session-lock.js", () => ({
|
||||
installPromptSubmissionLockRelease: hoisted.installPromptSubmissionLockRelease,
|
||||
}));
|
||||
|
||||
import { prepareEmbeddedAttemptPromptExecution } from "./attempt-prompt-execution-prepare.js";
|
||||
|
||||
@@ -53,14 +44,6 @@ function createInput(overrides: Partial<PromptExecutionInput> = {}): PromptExecu
|
||||
fsBridge: { readFile: vi.fn() },
|
||||
workspaceDir: "/sandbox/workspace",
|
||||
},
|
||||
session: { agent: { streamFn: vi.fn() } },
|
||||
sessionLockController: {
|
||||
canAdvanceSessionEntryCache: hoisted.canAdvanceSessionEntryCache,
|
||||
publishOwnedSessionFileSnapshot: hoisted.publishOwnedSessionFileSnapshot,
|
||||
reacquireAfterPrompt: hoisted.reacquireAfterPrompt,
|
||||
releaseForPrompt: hoisted.releaseForPrompt,
|
||||
withSessionWriteLock: hoisted.withSessionWriteLock,
|
||||
},
|
||||
skipPromptSubmission: false,
|
||||
...overrides,
|
||||
} as PromptExecutionInput;
|
||||
@@ -97,32 +80,14 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
|
||||
loadedCount: 0,
|
||||
skippedCount: 0,
|
||||
});
|
||||
expect(hoisted.installPromptSubmissionLockRelease).not.toHaveBeenCalled();
|
||||
expect(hoisted.detectAndLoadPromptImages).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("installs the lock handoff before loading prompt images", async () => {
|
||||
it("loads prompt images with the prepared workspace policy", async () => {
|
||||
const input = createInput();
|
||||
|
||||
const result = await prepareEmbeddedAttemptPromptExecution(input);
|
||||
|
||||
expect(hoisted.installPromptSubmissionLockRelease).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
session: input.session,
|
||||
sessionFile: "/tmp/session.jsonl",
|
||||
sessionKey: "agent:main:session-1",
|
||||
}),
|
||||
);
|
||||
const lockHandoff = hoisted.installPromptSubmissionLockRelease.mock.calls[0]?.[0] as
|
||||
| {
|
||||
reacquireAfterPrompt: () => Promise<void>;
|
||||
releaseForPrompt: () => Promise<void>;
|
||||
}
|
||||
| undefined;
|
||||
await lockHandoff?.releaseForPrompt();
|
||||
await lockHandoff?.reacquireAfterPrompt();
|
||||
expect(hoisted.releaseForPrompt).toHaveBeenCalledOnce();
|
||||
expect(hoisted.reacquireAfterPrompt).toHaveBeenCalledOnce();
|
||||
expect(hoisted.detectAndLoadPromptImages).toHaveBeenCalledWith({
|
||||
prompt: "inspect image.png",
|
||||
workspaceDir: "/tmp/workspace",
|
||||
@@ -152,7 +117,6 @@ describe("prepareEmbeddedAttemptPromptExecution", () => {
|
||||
|
||||
await prepareEmbeddedAttemptPromptExecution(input);
|
||||
|
||||
expect(hoisted.installPromptSubmissionLockRelease).toHaveBeenCalledOnce();
|
||||
expect(hoisted.detectAndLoadPromptImages).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ sandbox: undefined }),
|
||||
);
|
||||
|
||||
@@ -1,14 +1,8 @@
|
||||
/** Prepares prompt-lock ownership and prompt-local images for submission. */
|
||||
import { MAX_IMAGE_BYTES } from "@openclaw/media-core/constants";
|
||||
import type { OwnedSessionTranscriptCacheSnapshot } from "../../../config/sessions/transcript-write-context.js";
|
||||
import { readPersistedMediaFacts } from "../../../media/media-facts.js";
|
||||
import { resolveImageSanitizationLimits } from "../../image-sanitization.js";
|
||||
import type { SandboxContext } from "../../sandbox/types.js";
|
||||
import type { AgentSession } from "../../sessions/index.js";
|
||||
import {
|
||||
type EmbeddedAttemptSessionLockController,
|
||||
installPromptSubmissionLockRelease,
|
||||
} from "./attempt.session-lock.js";
|
||||
import { detectAndLoadPromptImages } from "./images.js";
|
||||
import { readPersistedMediaImageLayout } from "./prompt-image-metadata.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
@@ -44,8 +38,6 @@ export async function prepareEmbeddedAttemptPromptExecution(input: {
|
||||
effectiveWorkspace: string;
|
||||
prompt: string;
|
||||
sandbox?: SandboxContext | null;
|
||||
session: AgentSession;
|
||||
sessionLockController: EmbeddedAttemptSessionLockController;
|
||||
skipPromptSubmission: boolean;
|
||||
}): Promise<PromptImageResult> {
|
||||
if (input.skipPromptSubmission) {
|
||||
@@ -53,22 +45,6 @@ export async function prepareEmbeddedAttemptPromptExecution(input: {
|
||||
}
|
||||
|
||||
const { attempt } = input;
|
||||
installPromptSubmissionLockRelease({
|
||||
session: input.session,
|
||||
releaseForPrompt: () => input.sessionLockController.releaseForPrompt(),
|
||||
reacquireAfterPrompt: () => input.sessionLockController.reacquireAfterPrompt(),
|
||||
sessionKey: attempt.sessionKey,
|
||||
sessionFile: attempt.sessionFile,
|
||||
sessionTarget: attempt.sessionTarget,
|
||||
withSessionWriteLock: (run, options) =>
|
||||
input.sessionLockController.withSessionWriteLock(run, options),
|
||||
assertSessionWriteLockOwned: () => input.sessionLockController.assertOwned(),
|
||||
canAdvanceSessionEntryCache: (snapshot: OwnedSessionTranscriptCacheSnapshot) =>
|
||||
input.sessionLockController.canAdvanceSessionEntryCache(snapshot),
|
||||
publishSessionFileSnapshot: (snapshot: OwnedSessionTranscriptCacheSnapshot) =>
|
||||
input.sessionLockController.publishOwnedSessionFileSnapshot(snapshot),
|
||||
});
|
||||
|
||||
const persistedMessage =
|
||||
attempt.userTurnTranscriptRecorder?.message ??
|
||||
(await attempt.userTurnTranscriptRecorder?.resolveMessage());
|
||||
|
||||
@@ -164,7 +164,6 @@ function createFixture() {
|
||||
appendCustomEntry: vi.fn(),
|
||||
getEntries: vi.fn(() => []),
|
||||
};
|
||||
const sessionLockController = {};
|
||||
const input = {
|
||||
attempt: {
|
||||
model: { id: "model-1", provider: "test" },
|
||||
@@ -175,8 +174,7 @@ function createFixture() {
|
||||
},
|
||||
activeSession,
|
||||
sessionManager,
|
||||
sessionLockController,
|
||||
withOwnedSessionWriteLock: async <T>(operation: () => Promise<T> | T) => await operation(),
|
||||
withOwnedTranscriptWrite: async <T>(operation: () => Promise<T> | T) => await operation(),
|
||||
getCompactionReserveTokens: () => 77,
|
||||
assembly: {
|
||||
hookRunner: null,
|
||||
|
||||
@@ -41,7 +41,7 @@ type PromptContextPhaseInput = Omit<
|
||||
PromptContextInput,
|
||||
"attempt" | "messages" | "prompt" | "replaceSessionMessages"
|
||||
>;
|
||||
type PromptExecutionPhaseInput = Omit<PromptDispatchInput["execution"], "sessionLockController">;
|
||||
type PromptExecutionPhaseInput = PromptDispatchInput["execution"];
|
||||
type PromptObservationPhaseInput = Omit<PromptDispatchInput["observation"], "transcriptLeafId">;
|
||||
type PromptToolSurface = ReturnType<typeof applyPromptBuildToolsAllow>;
|
||||
type PromptPreflightPhaseInput = PromptDispatchInput["preflight"] & {
|
||||
@@ -54,14 +54,13 @@ type PromptSubmissionPhaseInput = Pick<
|
||||
| "toolResultPromptProjectionState"
|
||||
| "trajectoryRecorder"
|
||||
>;
|
||||
type WithOwnedSessionWriteLock = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
type WithOwnedTranscriptWrite = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
|
||||
export async function runEmbeddedAttemptPromptPhase(input: {
|
||||
attempt: PromptAssemblyInput["attempt"];
|
||||
activeSession: PromptAssemblyInput["activeSession"];
|
||||
sessionManager: PromptAssemblyInput["sessionManager"];
|
||||
sessionLockController: PromptDispatchInput["execution"]["sessionLockController"];
|
||||
withOwnedSessionWriteLock: WithOwnedSessionWriteLock;
|
||||
withOwnedTranscriptWrite: WithOwnedTranscriptWrite;
|
||||
getCompactionReserveTokens: () => number;
|
||||
emptyExplicitToolAllowlistError?: Error;
|
||||
assembly: PromptAssemblyPhaseInput;
|
||||
@@ -223,7 +222,7 @@ export async function runEmbeddedAttemptPromptPhase(input: {
|
||||
modelPrompt: promptForModel,
|
||||
sessionManager,
|
||||
systemPrompt: systemPromptForHook,
|
||||
withOwnedSessionWriteLock: input.withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite: input.withOwnedTranscriptWrite,
|
||||
});
|
||||
if (beforeAgentRunOutcome) {
|
||||
input.lifecycle.markBeforeAgentRunBlocked(beforeAgentRunOutcome);
|
||||
@@ -248,7 +247,7 @@ export async function runEmbeddedAttemptPromptPhase(input: {
|
||||
provider: attempt.provider,
|
||||
sessionManager: {
|
||||
appendCustomEntry: async (customType, data) => {
|
||||
await input.withOwnedSessionWriteLock(() => {
|
||||
await input.withOwnedTranscriptWrite(() => {
|
||||
sessionManager.appendCustomEntry(customType, data);
|
||||
});
|
||||
},
|
||||
@@ -276,10 +275,7 @@ export async function runEmbeddedAttemptPromptPhase(input: {
|
||||
...input.lifecycle.readState(),
|
||||
skipPromptSubmission,
|
||||
},
|
||||
execution: {
|
||||
...input.execution,
|
||||
sessionLockController: input.sessionLockController,
|
||||
},
|
||||
execution: input.execution,
|
||||
observation: {
|
||||
...input.observation,
|
||||
...(promptToolSurface
|
||||
@@ -313,8 +309,7 @@ export async function runEmbeddedAttemptPromptPhase(input: {
|
||||
handleMidTurnPrecheckRequest,
|
||||
markYieldAborted: input.lifecycle.markYieldAborted,
|
||||
releaseLeasedSteering,
|
||||
sessionLockController: input.sessionLockController,
|
||||
withOwnedSessionWriteLock: input.withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite: input.withOwnedTranscriptWrite,
|
||||
...input.lifecycle.readYieldState(),
|
||||
});
|
||||
if (promptErrorOutcome.promptFailure) {
|
||||
@@ -332,7 +327,7 @@ export async function runEmbeddedAttemptPromptPhase(input: {
|
||||
|
||||
const pendingMidTurnPrecheckRequest = input.lifecycle.takePendingMidTurnPrecheckRequest();
|
||||
if (pendingMidTurnPrecheckRequest) {
|
||||
await input.withOwnedSessionWriteLock(() => {
|
||||
await input.withOwnedTranscriptWrite(() => {
|
||||
removeTrailingMidTurnPrecheckAssistantError({ activeSession, sessionManager });
|
||||
const state = input.lifecycle.readState();
|
||||
if (!state.preflightRecovery && state.promptErrorSource !== "precheck") {
|
||||
|
||||
@@ -29,9 +29,9 @@ const attempt = {
|
||||
} as never;
|
||||
|
||||
function createInput(overrides: Record<string, unknown> = {}) {
|
||||
const sessionLockController = {
|
||||
acquireForCleanup: vi.fn(async () => ({ release: vi.fn() })),
|
||||
hasSessionTakeover: vi.fn(() => false),
|
||||
const transcriptLifecycle = {
|
||||
beginCleanup: vi.fn(async () => undefined),
|
||||
dispose: vi.fn(async () => undefined),
|
||||
};
|
||||
const emitDiagnosticRunCompleted = vi.fn();
|
||||
const trajectoryRecorder = {
|
||||
@@ -52,7 +52,7 @@ function createInput(overrides: Record<string, unknown> = {}) {
|
||||
};
|
||||
return {
|
||||
attempt,
|
||||
sessionLockController,
|
||||
transcriptLifecycle,
|
||||
sessionAgentId: "main",
|
||||
buildAbortSettlePromise: () => null,
|
||||
trajectoryRecorder,
|
||||
@@ -71,7 +71,7 @@ describe("cleanupEmbeddedAttemptSessionPhase", () => {
|
||||
hoisted.flushEmbeddedAttemptTrajectoryRecorder.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it("records the terminal event before lock-safe resource cleanup", async () => {
|
||||
it("records the terminal event before transcript-safe resource cleanup", async () => {
|
||||
const input = createInput();
|
||||
|
||||
await cleanupEmbeddedAttemptSessionPhase(input as never);
|
||||
@@ -91,8 +91,10 @@ describe("cleanupEmbeddedAttemptSessionPhase", () => {
|
||||
expect.objectContaining({ runId: "run-1", sessionId: "session-1", agentId: "main" }),
|
||||
);
|
||||
expect(hoisted.cleanupEmbeddedAttemptResources).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ aborted: false, skipSessionFlush: false }),
|
||||
expect.objectContaining({ aborted: false }),
|
||||
);
|
||||
expect(input.transcriptLifecycle.beginCleanup).toHaveBeenCalledOnce();
|
||||
expect(input.transcriptLifecycle.dispose).toHaveBeenCalledOnce();
|
||||
expect(input.emitDiagnosticRunCompleted).toHaveBeenCalledWith("completed", null, undefined);
|
||||
});
|
||||
|
||||
@@ -126,38 +128,4 @@ describe("cleanupEmbeddedAttemptSessionPhase", () => {
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves the prompt error when cleanup detects session takeover", async () => {
|
||||
const promptError = new Error("prompt failed");
|
||||
const sessionLockController = {
|
||||
acquireForCleanup: vi.fn(async () => ({ release: vi.fn() })),
|
||||
hasSessionTakeover: vi.fn(() => true),
|
||||
};
|
||||
const emitDiagnosticRunCompleted = vi.fn();
|
||||
const input = createInput({
|
||||
sessionLockController,
|
||||
emitDiagnosticRunCompleted,
|
||||
trajectoryRecorder: null,
|
||||
readState: () => ({
|
||||
aborted: false,
|
||||
externalAbort: false,
|
||||
timedOut: false,
|
||||
idleTimedOut: false,
|
||||
timedOutDuringCompaction: false,
|
||||
timedOutDuringToolExecution: false,
|
||||
timedOutByRunBudget: false,
|
||||
promptError,
|
||||
beforeAgentRunBlocked: false,
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(cleanupEmbeddedAttemptSessionPhase(input as never)).rejects.toMatchObject({
|
||||
name: "EmbeddedAttemptSessionTakeoverError",
|
||||
promptError,
|
||||
});
|
||||
expect(hoisted.cleanupEmbeddedAttemptResources).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ skipSessionFlush: true }),
|
||||
);
|
||||
expect(emitDiagnosticRunCompleted).toHaveBeenCalledWith("error", promptError, undefined);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,16 +10,11 @@ import { log } from "../logger.js";
|
||||
import { flushPendingToolResultsAfterIdle } from "../wait-for-idle-before-flush.js";
|
||||
import type { EmitDiagnosticRunCompleted } from "./attempt-startup.js";
|
||||
import { flushEmbeddedAttemptTrajectoryRecorder } from "./attempt-trajectory-flush-cleanup.js";
|
||||
import {
|
||||
type createEmbeddedAttemptSessionLockController,
|
||||
EmbeddedAttemptSessionTakeoverError,
|
||||
} from "./attempt.session-lock.js";
|
||||
import type { createEmbeddedAttemptTranscriptLifecycle } from "./attempt-transcript-lifecycle.js";
|
||||
import { cleanupEmbeddedAttemptResources } from "./attempt.subscription-cleanup.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type AttemptSessionLockController = Awaited<
|
||||
ReturnType<typeof createEmbeddedAttemptSessionLockController>
|
||||
>;
|
||||
type AttemptTranscriptLifecycle = ReturnType<typeof createEmbeddedAttemptTranscriptLifecycle>;
|
||||
type TrajectoryRecorder = ReturnType<typeof createTrajectoryRuntimeRecorder>;
|
||||
type DisposableRuntime = { dispose(): Promise<void> | void };
|
||||
|
||||
@@ -27,7 +22,7 @@ type CleanupEmbeddedAttemptSessionInput = {
|
||||
attempt: EmbeddedRunAttemptParams;
|
||||
session?: AgentSession;
|
||||
sessionManager?: ReturnType<typeof guardSessionManager>;
|
||||
sessionLockController: AttemptSessionLockController;
|
||||
transcriptLifecycle: AttemptTranscriptLifecycle;
|
||||
bundleMcpRuntime?: DisposableRuntime;
|
||||
bundleLspRuntime?: DisposableRuntime;
|
||||
removeToolResultContextGuard?: () => void;
|
||||
@@ -53,28 +48,6 @@ type CleanupEmbeddedAttemptSessionInput = {
|
||||
};
|
||||
};
|
||||
|
||||
class EmbeddedAttemptPromptErrorWithCleanupTakeoverError extends Error {
|
||||
readonly promptError: unknown;
|
||||
readonly cleanupError: EmbeddedAttemptSessionTakeoverError;
|
||||
|
||||
constructor(params: { promptError: unknown; cleanupError: EmbeddedAttemptSessionTakeoverError }) {
|
||||
super(formatErrorMessage(params.promptError), { cause: params.cleanupError });
|
||||
this.name = "EmbeddedAttemptSessionTakeoverError";
|
||||
this.promptError = params.promptError;
|
||||
this.cleanupError = params.cleanupError;
|
||||
}
|
||||
}
|
||||
|
||||
function shouldPreservePromptErrorAfterCleanupError(params: {
|
||||
promptError: unknown;
|
||||
cleanupError: unknown;
|
||||
}): boolean {
|
||||
return (
|
||||
Boolean(params.promptError) &&
|
||||
params.cleanupError instanceof EmbeddedAttemptSessionTakeoverError
|
||||
);
|
||||
}
|
||||
|
||||
export async function cleanupEmbeddedAttemptSessionPhase(
|
||||
input: CleanupEmbeddedAttemptSessionInput,
|
||||
): Promise<void> {
|
||||
@@ -128,9 +101,7 @@ export async function cleanupEmbeddedAttemptSessionPhase(
|
||||
cleanupState.idleTimedOut ||
|
||||
cleanupState.timedOutDuringCompaction;
|
||||
const cleanupAbortLike = cleanupAborted || input.cleanupYieldAborted;
|
||||
const cleanupSessionLock = await input.sessionLockController.acquireForCleanup({
|
||||
session: input.session,
|
||||
});
|
||||
await input.transcriptLifecycle.beginCleanup();
|
||||
await cleanupEmbeddedAttemptResources({
|
||||
removeToolResultContextGuard: input.removeToolResultContextGuard,
|
||||
flushPendingToolResultsAfterIdle,
|
||||
@@ -138,28 +109,24 @@ export async function cleanupEmbeddedAttemptSessionPhase(
|
||||
sessionManager: input.sessionManager,
|
||||
bundleMcpRuntime: input.bundleMcpRuntime,
|
||||
bundleLspRuntime: input.bundleLspRuntime,
|
||||
sessionLock: cleanupSessionLock,
|
||||
// Aborted runs skip the idle wait so teardown cannot strand the lock.
|
||||
aborted: cleanupAbortLike,
|
||||
abortSettlePromise: cleanupAborted ? input.buildAbortSettlePromise() : null,
|
||||
skipSessionFlush: input.sessionLockController.hasSessionTakeover(),
|
||||
runId: attempt.runId,
|
||||
sessionId: attempt.sessionId,
|
||||
});
|
||||
} catch (err) {
|
||||
cleanupError = err;
|
||||
} finally {
|
||||
try {
|
||||
await input.transcriptLifecycle.dispose();
|
||||
} catch (err) {
|
||||
cleanupError ??= err;
|
||||
}
|
||||
}
|
||||
|
||||
const finalState = input.readState();
|
||||
const synthesizedCleanupTakeoverError =
|
||||
!cleanupError && finalState.promptError && input.sessionLockController.hasSessionTakeover()
|
||||
? new EmbeddedAttemptSessionTakeoverError(attempt.sessionFile)
|
||||
: undefined;
|
||||
const cleanupFailure = cleanupError ?? synthesizedCleanupTakeoverError;
|
||||
const shouldPreservePromptError = shouldPreservePromptErrorAfterCleanupError({
|
||||
promptError: finalState.promptError,
|
||||
cleanupError: cleanupFailure,
|
||||
});
|
||||
const cleanupFailure = cleanupError;
|
||||
input.emitDiagnosticRunCompleted?.(
|
||||
cleanupFailure
|
||||
? "error"
|
||||
@@ -173,7 +140,7 @@ export async function cleanupEmbeddedAttemptSessionPhase(
|
||||
finalState.timedOutDuringCompaction
|
||||
? "aborted"
|
||||
: "completed",
|
||||
shouldPreservePromptError ? finalState.promptError : (cleanupFailure ?? finalState.promptError),
|
||||
cleanupFailure ?? finalState.promptError,
|
||||
finalState.beforeAgentRunBlocked
|
||||
? { blockedBy: finalState.beforeAgentRunBlockedBy ?? "before_agent_run" }
|
||||
: undefined,
|
||||
@@ -182,18 +149,5 @@ export async function cleanupEmbeddedAttemptSessionPhase(
|
||||
if (!cleanupFailure) {
|
||||
return;
|
||||
}
|
||||
if (shouldPreservePromptError) {
|
||||
log.warn(
|
||||
`embedded attempt cleanup detected session takeover after prompt failure; preserving prompt error: ` +
|
||||
`runId=${attempt.runId} sessionId=${attempt.sessionId} ` +
|
||||
`promptError=${formatErrorMessage(finalState.promptError)} cleanupError=${formatErrorMessage(cleanupFailure)}`,
|
||||
);
|
||||
await Promise.reject(
|
||||
new EmbeddedAttemptPromptErrorWithCleanupTakeoverError({
|
||||
promptError: finalState.promptError,
|
||||
cleanupError: cleanupFailure as EmbeddedAttemptSessionTakeoverError,
|
||||
}),
|
||||
);
|
||||
}
|
||||
await Promise.reject(toErrorObject(cleanupFailure, "Non-Error rejection"));
|
||||
}
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
acquireSessionFileOwner: vi.fn(),
|
||||
acquireSessionWriteLock: vi.fn(),
|
||||
createSessionLockController: vi.fn(),
|
||||
resolveSessionWriteLockTargetKey: vi.fn(() => "canonical-lock-key"),
|
||||
resolveCompactionTimeoutMs: vi.fn(() => 30_000),
|
||||
resolveSessionWriteLockOptions: vi.fn(() => ({
|
||||
timeoutMs: 100,
|
||||
staleMs: 200,
|
||||
maxHoldMs: 300,
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../../session-write-lock.js", () => ({
|
||||
acquireSessionWriteLock: mocks.acquireSessionWriteLock,
|
||||
resolveSessionWriteLockTargetKey: mocks.resolveSessionWriteLockTargetKey,
|
||||
}));
|
||||
vi.mock("../compaction-safety-timeout.js", () => ({
|
||||
resolveCompactionTimeoutMs: mocks.resolveCompactionTimeoutMs,
|
||||
}));
|
||||
vi.mock("./attempt.run-decisions.js", () => ({
|
||||
resolveEmbeddedAttemptSessionWriteLockOptions: mocks.resolveSessionWriteLockOptions,
|
||||
}));
|
||||
vi.mock("./attempt.session-lock.js", () => ({
|
||||
acquireEmbeddedAttemptSessionFileOwner: mocks.acquireSessionFileOwner,
|
||||
createEmbeddedAttemptSessionLockController: mocks.createSessionLockController,
|
||||
}));
|
||||
|
||||
import { prepareEmbeddedAttemptSessionLock } from "./attempt-session-lock-prepare.js";
|
||||
|
||||
type PrepareInput = Parameters<typeof prepareEmbeddedAttemptSessionLock>[0];
|
||||
type CreateControllerInput = {
|
||||
acquireSessionWriteLock: unknown;
|
||||
initialAcquireSignal?: AbortSignal;
|
||||
lockOptions: {
|
||||
sessionFile: string;
|
||||
timeoutMs: number;
|
||||
staleMs: number;
|
||||
maxHoldMs: number;
|
||||
};
|
||||
reloadPromptReleasedSessionFile: () => void;
|
||||
};
|
||||
|
||||
function createFixture(options?: { rejectPostArmFence?: Error }) {
|
||||
const order: string[] = [];
|
||||
const sessionFileOwner = { release: vi.fn() };
|
||||
const sessionManager = {
|
||||
reloadPersistedTranscript: vi.fn(),
|
||||
};
|
||||
const sessionLockController = {
|
||||
assertOwned: vi.fn(),
|
||||
canAdvanceSessionEntryCache: vi.fn(() => true),
|
||||
dispose: vi.fn(async () => undefined),
|
||||
publishOwnedSessionFileSnapshot: vi.fn(() => true),
|
||||
withSessionWriteLock: vi.fn(async <T>(operation: () => Promise<T> | T) => await operation()),
|
||||
};
|
||||
mocks.acquireSessionFileOwner.mockImplementation(async () => {
|
||||
order.push("file-owner");
|
||||
return sessionFileOwner;
|
||||
});
|
||||
mocks.createSessionLockController.mockImplementation(async () => {
|
||||
order.push("session-lock");
|
||||
return sessionLockController;
|
||||
});
|
||||
|
||||
let fenceCount = 0;
|
||||
const externalAbortController = {
|
||||
arm: vi.fn(() => {
|
||||
order.push("arm");
|
||||
}),
|
||||
throwIfFiredAfterPrepCleanup: vi.fn(async () => {
|
||||
fenceCount += 1;
|
||||
order.push(`abort-fence-${fenceCount}`);
|
||||
if (fenceCount === 2 && options?.rejectPostArmFence) {
|
||||
throw options.rejectPostArmFence;
|
||||
}
|
||||
}),
|
||||
};
|
||||
let releaseSessionLock: (() => Promise<void>) | undefined;
|
||||
const input = {
|
||||
attempt: {
|
||||
abortSignal: new AbortController().signal,
|
||||
config: {},
|
||||
sessionFile: "/tmp/session.jsonl",
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:main:session-1",
|
||||
sessionTarget: {
|
||||
agentId: "main",
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:main:session-1",
|
||||
storePath: "/tmp/sessions.json",
|
||||
},
|
||||
},
|
||||
externalAbortController,
|
||||
getSessionManager: () => sessionManager,
|
||||
onSessionFileOwnerAcquired: vi.fn(() => {
|
||||
order.push("publish-file-owner");
|
||||
}),
|
||||
onSessionLockReleaseReady: vi.fn((release) => {
|
||||
order.push("publish-session-lock");
|
||||
releaseSessionLock = release;
|
||||
}),
|
||||
} as unknown as PrepareInput;
|
||||
|
||||
return {
|
||||
externalAbortController,
|
||||
input,
|
||||
order,
|
||||
releaseSessionLock: () => releaseSessionLock,
|
||||
sessionFileOwner,
|
||||
sessionLockController,
|
||||
sessionManager,
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("prepareEmbeddedAttemptSessionLock", () => {
|
||||
it("publishes ownership before arming and exposes owned transcript writes", async () => {
|
||||
const fixture = createFixture();
|
||||
|
||||
const result = await prepareEmbeddedAttemptSessionLock(fixture.input);
|
||||
|
||||
expect(fixture.order).toEqual([
|
||||
"abort-fence-1",
|
||||
"file-owner",
|
||||
"publish-file-owner",
|
||||
"session-lock",
|
||||
"publish-session-lock",
|
||||
"arm",
|
||||
"abort-fence-2",
|
||||
]);
|
||||
expect(result.compactionTimeoutMs).toBe(30_000);
|
||||
expect(mocks.acquireSessionFileOwner).toHaveBeenCalledWith({
|
||||
sessionFile: "/tmp/session.jsonl",
|
||||
timeoutMs: 300,
|
||||
signal: fixture.input.attempt.abortSignal,
|
||||
});
|
||||
|
||||
const controllerInput = mocks.createSessionLockController.mock.calls[0]?.[0] as
|
||||
| CreateControllerInput
|
||||
| undefined;
|
||||
expect(controllerInput).toEqual(
|
||||
expect.objectContaining({
|
||||
acquireSessionWriteLock: mocks.acquireSessionWriteLock,
|
||||
initialAcquireSignal: fixture.input.attempt.abortSignal,
|
||||
lockOptions: {
|
||||
sessionFile: "canonical-lock-key",
|
||||
targetKind: "session-key",
|
||||
timeoutMs: 100,
|
||||
staleMs: 200,
|
||||
maxHoldMs: 300,
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(mocks.resolveSessionWriteLockTargetKey).toHaveBeenCalledWith(
|
||||
fixture.input.attempt.sessionTarget,
|
||||
);
|
||||
controllerInput?.reloadPromptReleasedSessionFile();
|
||||
expect(fixture.sessionManager.reloadPersistedTranscript).toHaveBeenCalledOnce();
|
||||
|
||||
await expect(result.withOwnedSessionWriteLock(async () => "done")).resolves.toBe("done");
|
||||
expect(fixture.sessionLockController.withSessionWriteLock).toHaveBeenCalledOnce();
|
||||
await fixture.releaseSessionLock()?.();
|
||||
expect(fixture.sessionLockController.dispose).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("publishes both cleanup handles before a post-arm abort fence fails", async () => {
|
||||
const abortError = new Error("aborted during lock acquisition");
|
||||
const fixture = createFixture({ rejectPostArmFence: abortError });
|
||||
|
||||
await expect(prepareEmbeddedAttemptSessionLock(fixture.input)).rejects.toBe(abortError);
|
||||
|
||||
expect(fixture.input.onSessionFileOwnerAcquired).toHaveBeenCalledWith(fixture.sessionFileOwner);
|
||||
expect(fixture.input.onSessionLockReleaseReady).toHaveBeenCalledOnce();
|
||||
expect(fixture.externalAbortController.arm).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -1,125 +0,0 @@
|
||||
/** Acquires and publishes the session-write ownership used by one attempt. */
|
||||
import { withOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js";
|
||||
import { resolveAgentRunSessionTarget } from "../../run-session-target.js";
|
||||
import type { guardSessionManager } from "../../session-tool-result-guard-wrapper.js";
|
||||
import {
|
||||
acquireSessionWriteLock,
|
||||
resolveSessionWriteLockTargetKey,
|
||||
} from "../../session-write-lock.js";
|
||||
import { resolveCompactionTimeoutMs } from "../compaction-safety-timeout.js";
|
||||
import { resolveEmbeddedAttemptSessionWriteLockOptions } from "./attempt.run-decisions.js";
|
||||
import {
|
||||
acquireEmbeddedAttemptSessionFileOwner,
|
||||
type EmbeddedAttemptSessionFileOwner,
|
||||
createEmbeddedAttemptSessionLockController,
|
||||
} from "./attempt.session-lock.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type AttemptSessionLockController = Awaited<
|
||||
ReturnType<typeof createEmbeddedAttemptSessionLockController>
|
||||
>;
|
||||
type OwnedTranscriptWriteContext = Parameters<typeof withOwnedSessionTranscriptWrites>[0];
|
||||
type WithOwnedSessionWriteLock = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
|
||||
export async function prepareEmbeddedAttemptSessionLock(input: {
|
||||
attempt: Pick<
|
||||
EmbeddedRunAttemptParams,
|
||||
| "abortSignal"
|
||||
| "config"
|
||||
| "runId"
|
||||
| "sessionFile"
|
||||
| "sessionId"
|
||||
| "sessionKey"
|
||||
| "sessionTarget"
|
||||
>;
|
||||
externalAbortController: {
|
||||
arm: () => void;
|
||||
throwIfFiredAfterPrepCleanup: () => Promise<void>;
|
||||
};
|
||||
getSessionManager: () => ReturnType<typeof guardSessionManager> | undefined;
|
||||
onSessionFileOwnerAcquired: (owner: EmbeddedAttemptSessionFileOwner) => void;
|
||||
onSessionLockReleaseReady: (release: () => Promise<void>) => void;
|
||||
}): Promise<{
|
||||
compactionTimeoutMs: number;
|
||||
ownedTranscriptWriteContext: OwnedTranscriptWriteContext;
|
||||
sessionLockController: AttemptSessionLockController;
|
||||
withOwnedSessionWriteLock: WithOwnedSessionWriteLock;
|
||||
}> {
|
||||
const { attempt, externalAbortController } = input;
|
||||
const compactionTimeoutMs = resolveCompactionTimeoutMs(attempt.config);
|
||||
const sessionWriteLockOptions = resolveEmbeddedAttemptSessionWriteLockOptions({
|
||||
config: attempt.config,
|
||||
compactionTimeoutMs,
|
||||
});
|
||||
const sessionTarget = await resolveAgentRunSessionTarget({
|
||||
agentId: attempt.sessionTarget?.agentId,
|
||||
config: attempt.config,
|
||||
missingSessionKey: "resolve-existing",
|
||||
sessionFile: attempt.sessionFile,
|
||||
sessionId: attempt.sessionId,
|
||||
sessionKey: attempt.sessionKey,
|
||||
sessionTarget: attempt.sessionTarget,
|
||||
});
|
||||
|
||||
await externalAbortController.throwIfFiredAfterPrepCleanup();
|
||||
const sessionFileOwner = await acquireEmbeddedAttemptSessionFileOwner({
|
||||
sessionFile: attempt.sessionFile,
|
||||
timeoutMs: sessionWriteLockOptions.maxHoldMs,
|
||||
signal: attempt.abortSignal,
|
||||
});
|
||||
// Publish ownership immediately so outer teardown can release it if later
|
||||
// controller setup or the post-arm abort fence fails.
|
||||
input.onSessionFileOwnerAcquired(sessionFileOwner);
|
||||
|
||||
const getSessionManager = () => {
|
||||
const sessionManager = input.getSessionManager();
|
||||
if (!sessionManager) {
|
||||
throw new Error("session manager unavailable during prompt-released file reload");
|
||||
}
|
||||
return sessionManager;
|
||||
};
|
||||
const sessionLockController = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock,
|
||||
initialAcquireSignal: attempt.abortSignal,
|
||||
runId: attempt.runId,
|
||||
sessionId: attempt.sessionId,
|
||||
lockOptions: {
|
||||
sessionFile: resolveSessionWriteLockTargetKey(sessionTarget),
|
||||
targetKind: "session-key",
|
||||
...sessionWriteLockOptions,
|
||||
},
|
||||
reloadPromptReleasedSessionFile: () => {
|
||||
getSessionManager().reloadPersistedTranscript();
|
||||
},
|
||||
});
|
||||
input.onSessionLockReleaseReady(() => sessionLockController.dispose());
|
||||
|
||||
const ownedTranscriptWriteContext: OwnedTranscriptWriteContext = {
|
||||
sessionFile: attempt.sessionFile,
|
||||
sessionKey: attempt.sessionKey,
|
||||
sessionTarget: attempt.sessionTarget,
|
||||
assertOwned: () => sessionLockController.assertOwned(),
|
||||
canAdvanceSessionEntryCache: (snapshot) =>
|
||||
sessionLockController.canAdvanceSessionEntryCache(snapshot),
|
||||
publishSessionFileSnapshot: (snapshot) =>
|
||||
sessionLockController.publishOwnedSessionFileSnapshot(snapshot),
|
||||
withSessionWriteLock: (operation, options) =>
|
||||
sessionLockController.withSessionWriteLock(operation, options),
|
||||
};
|
||||
const withOwnedSessionWriteLock: WithOwnedSessionWriteLock = (operation) =>
|
||||
withOwnedSessionTranscriptWrites(ownedTranscriptWriteContext, async () =>
|
||||
sessionLockController.withSessionWriteLock(operation),
|
||||
);
|
||||
|
||||
externalAbortController.arm();
|
||||
// The signal can fire while the eager lock is acquired. Recheck after arming
|
||||
// so a stopped run never reaches session creation or provider prompt.
|
||||
await externalAbortController.throwIfFiredAfterPrepCleanup();
|
||||
|
||||
return {
|
||||
compactionTimeoutMs,
|
||||
ownedTranscriptWriteContext,
|
||||
sessionLockController,
|
||||
withOwnedSessionWriteLock,
|
||||
};
|
||||
}
|
||||
@@ -9,19 +9,19 @@ import { SessionManager } from "../../sessions/index.js";
|
||||
import { runContextEngineMaintenance } from "../context-engine-maintenance.js";
|
||||
import { log } from "../logger.js";
|
||||
import { resolveExistingAttemptTranscriptState } from "./attempt-transcript-helpers.js";
|
||||
import type { EmbeddedAttemptTranscriptLifecycle } from "./attempt-transcript-lifecycle.js";
|
||||
import {
|
||||
runAttemptContextEngineBootstrap,
|
||||
type AttemptContextEngine,
|
||||
} from "./attempt.context-engine-helpers.js";
|
||||
import { buildAfterTurnRuntimeContext } from "./attempt.prompt-helpers.js";
|
||||
import type { EmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
import { resolveAttemptTranscriptPolicy } from "./attempt.transcript-policy.js";
|
||||
import { createUserTranscriptContextRegistry } from "./attempt.user-transcript-context-registry.js";
|
||||
import { resolveSessionBoundaryPromptCacheKey } from "./session-boundary-prompt-cache-key.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type AttemptSessionManager = ReturnType<typeof guardSessionManager>;
|
||||
type WithOwnedSessionWriteLock = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
type WithOwnedTranscriptWrite = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
export async function prepareEmbeddedAttemptSessionManager(input: {
|
||||
attempt: EmbeddedRunAttemptParams;
|
||||
activeContextEngine?: AttemptContextEngine;
|
||||
@@ -32,8 +32,8 @@ export async function prepareEmbeddedAttemptSessionManager(input: {
|
||||
replayAllowedToolNames: ReadonlySet<string>;
|
||||
resolveActiveContextEnginePluginId: () => string | undefined;
|
||||
sessionAgentId: string;
|
||||
sessionLockController: EmbeddedAttemptSessionLockController;
|
||||
withOwnedSessionWriteLock: WithOwnedSessionWriteLock;
|
||||
transcriptLifecycle: EmbeddedAttemptTranscriptLifecycle;
|
||||
withOwnedTranscriptWrite: WithOwnedTranscriptWrite;
|
||||
}) {
|
||||
const { attempt } = input;
|
||||
const transcriptState = await resolveExistingAttemptTranscriptState({
|
||||
@@ -95,9 +95,6 @@ export async function prepareEmbeddedAttemptSessionManager(input: {
|
||||
attempt.suppressTranscriptOnlyAssistantPersistence,
|
||||
suppressAssistantErrorPersistence: attempt.suppressAssistantErrorPersistence,
|
||||
skipBeforeMessageWriteHooks: attempt.operation === "settled-tool-finalization",
|
||||
onMessagePersisted: () => {
|
||||
input.sessionLockController.refreshAfterOwnedSessionWrite();
|
||||
},
|
||||
onUserMessagePreparingForPersistence: (_message, recorder) => {
|
||||
latestPersistedUserMessage = undefined;
|
||||
latestUserTurnTranscriptRecorder = recorder;
|
||||
@@ -131,7 +128,7 @@ export async function prepareEmbeddedAttemptSessionManager(input: {
|
||||
// even when a context-engine or transcript preparation step fails.
|
||||
input.onSessionManagerCreated(sessionManager);
|
||||
|
||||
await input.withOwnedSessionWriteLock(async () => {
|
||||
await input.withOwnedTranscriptWrite(async () => {
|
||||
await runAttemptContextEngineBootstrap({
|
||||
hadSessionFile: transcriptState.hasBootstrapTranscriptState,
|
||||
contextEngine: input.activeContextEngine,
|
||||
|
||||
@@ -171,8 +171,8 @@ function createFixture() {
|
||||
replayAllowedToolNames: new Set(["read"]),
|
||||
resolveActiveContextEnginePluginId: vi.fn(),
|
||||
sessionAgentId: "main",
|
||||
sessionLockController: {},
|
||||
withOwnedSessionWriteLock: vi.fn(),
|
||||
transcriptLifecycle: {},
|
||||
withOwnedTranscriptWrite: vi.fn(),
|
||||
},
|
||||
agentSession: {
|
||||
agentCoreThinkingLevel: "medium",
|
||||
|
||||
@@ -48,8 +48,8 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
|
||||
| "replayAllowedToolNames"
|
||||
| "resolveActiveContextEnginePluginId"
|
||||
| "sessionAgentId"
|
||||
| "sessionLockController"
|
||||
| "withOwnedSessionWriteLock"
|
||||
| "transcriptLifecycle"
|
||||
| "withOwnedTranscriptWrite"
|
||||
>;
|
||||
agentSession: Pick<
|
||||
AgentSessionInput,
|
||||
@@ -99,8 +99,8 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
|
||||
replayAllowedToolNames: input.sessionManager.replayAllowedToolNames,
|
||||
resolveActiveContextEnginePluginId: input.sessionManager.resolveActiveContextEnginePluginId,
|
||||
sessionAgentId: input.sessionManager.sessionAgentId,
|
||||
sessionLockController: input.sessionManager.sessionLockController,
|
||||
withOwnedSessionWriteLock: input.sessionManager.withOwnedSessionWriteLock,
|
||||
transcriptLifecycle: input.sessionManager.transcriptLifecycle,
|
||||
withOwnedTranscriptWrite: input.sessionManager.withOwnedTranscriptWrite,
|
||||
});
|
||||
const { isOpenAIResponsesApi, preparedUserTurnMessage, sessionManager, transcriptPolicy } =
|
||||
preparedSessionManager;
|
||||
@@ -129,7 +129,7 @@ export async function prepareEmbeddedAttemptSessionRuntime(input: {
|
||||
},
|
||||
runAbortSignal: input.agentSession.runAbortSignal,
|
||||
sessionAgentId: input.sessionManager.sessionAgentId,
|
||||
sessionLockController: input.sessionManager.sessionLockController,
|
||||
transcriptLifecycle: input.sessionManager.transcriptLifecycle,
|
||||
sessionManager,
|
||||
});
|
||||
const { activeSession, setActiveSessionSystemPrompt, settingsManager } = preparedAgentSession;
|
||||
|
||||
@@ -109,8 +109,8 @@ function createInput(options?: {
|
||||
setActiveToolsByName,
|
||||
} as unknown as AgentSession;
|
||||
const sessionManager = { id: "session-manager" };
|
||||
const sessionLockController = {
|
||||
withSessionWriteLock: vi.fn(async (operation: () => unknown) => await operation()),
|
||||
const transcriptLifecycle = {
|
||||
withTranscriptWrite: vi.fn(async (operation: () => unknown) => await operation()),
|
||||
};
|
||||
const hookRunner = { id: "hooks" };
|
||||
const sessionToolAllowlist = [{ name: "read" }];
|
||||
@@ -181,7 +181,7 @@ function createInput(options?: {
|
||||
},
|
||||
runAbortSignal: new AbortController().signal,
|
||||
sessionAgentId: "agent-1",
|
||||
sessionLockController: sessionLockController as never,
|
||||
transcriptLifecycle: transcriptLifecycle as never,
|
||||
sessionManager: sessionManager as never,
|
||||
},
|
||||
onDeliveredSourceReply: () => onDeliveredSourceReply?.(),
|
||||
|
||||
@@ -21,8 +21,8 @@ import { log } from "../logger.js";
|
||||
import { createEmbeddedAgentResourceLoader } from "../resource-loader.js";
|
||||
import { applySystemPromptToSession } from "../system-prompt.js";
|
||||
import { prepareEmbeddedAttemptClientTools } from "./attempt-client-tools.js";
|
||||
import type { EmbeddedAttemptTranscriptLifecycle } from "./attempt-transcript-lifecycle.js";
|
||||
import type { AttemptContextEngine } from "./attempt.context-engine-helpers.js";
|
||||
import type { EmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
import { installCodeModeRepairHook } from "./code-mode-repair.js";
|
||||
import { installMessageToolOnlyTerminalHook } from "./message-tool-terminal.js";
|
||||
import { notifyToolActivity } from "./tool-activity-heartbeat.js";
|
||||
@@ -54,7 +54,7 @@ export async function prepareEmbeddedAttemptAgentSession(input: {
|
||||
onSystemPromptChanged: (systemPrompt: string) => void;
|
||||
runAbortSignal: AbortSignal;
|
||||
sessionAgentId: string;
|
||||
sessionLockController: EmbeddedAttemptSessionLockController;
|
||||
transcriptLifecycle: EmbeddedAttemptTranscriptLifecycle;
|
||||
sessionManager: AttemptSessionManager;
|
||||
}) {
|
||||
const { attempt } = input;
|
||||
@@ -165,8 +165,8 @@ export async function prepareEmbeddedAttemptAgentSession(input: {
|
||||
return hydratedTool;
|
||||
}
|
||||
: undefined,
|
||||
withSessionWriteLock: (operation) =>
|
||||
input.sessionLockController.withSessionWriteLock(operation),
|
||||
withSessionWriteSettlement: (operation) =>
|
||||
input.transcriptLifecycle.withTranscriptWrite(operation),
|
||||
};
|
||||
const createdSession = await createAgentSessionForEmbeddedRunner(sessionOptions, {
|
||||
// Without a resolved model budget, the outer loop cannot own bounded recovery.
|
||||
|
||||
@@ -43,13 +43,7 @@ function createFixture(overrides?: Partial<FinalizeInput>) {
|
||||
sessionManager: {
|
||||
buildSessionContext: () => ({ messages: repairedMessages }),
|
||||
},
|
||||
sessionLockController: {
|
||||
releaseForPrompt: vi.fn(async () => {
|
||||
order.push("release-prompt-lock");
|
||||
}),
|
||||
isPromptSubmissionBlockedError: vi.fn(() => false),
|
||||
},
|
||||
withOwnedSessionWriteLock: vi.fn(async (operation) => await operation()),
|
||||
withOwnedTranscriptWrite: vi.fn(async (operation) => await operation()),
|
||||
waitForPendingEvents: vi.fn(async () => {
|
||||
order.push("pending-events");
|
||||
}),
|
||||
@@ -278,13 +272,7 @@ describe("finalizeEmbeddedAttemptStreamPhase", () => {
|
||||
});
|
||||
|
||||
expect(fixture.activeSession.agent.state.messages).toBe(fixture.repairedMessages);
|
||||
expect(fixture.order).toEqual([
|
||||
"pending-events",
|
||||
"release-prompt-lock",
|
||||
"settle",
|
||||
"settled-published",
|
||||
"after-turn",
|
||||
]);
|
||||
expect(fixture.order).toEqual(["pending-events", "settle", "settled-published", "after-turn"]);
|
||||
expect(mocks.settleStream).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runAbortDeadlineAtMs: 123,
|
||||
@@ -387,17 +375,9 @@ describe("finalizeEmbeddedAttemptStreamPhase", () => {
|
||||
expect(mocks.settleStream).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("settles an aborted run when prompt release returns its recorded cancellation reason", async () => {
|
||||
it("settles an aborted run with its recorded cancellation reason", async () => {
|
||||
const cancellationReason = new Error("cancelled by operator");
|
||||
const fixture = createFixture({
|
||||
sessionLockController: {
|
||||
releaseForPrompt: vi.fn(async () => {
|
||||
throw cancellationReason;
|
||||
}),
|
||||
isPromptSubmissionBlockedError: (error: unknown) => error === cancellationReason,
|
||||
} as never,
|
||||
repairedRejectedThinkingReplay: false,
|
||||
});
|
||||
const fixture = createFixture({ repairedRejectedThinkingReplay: false });
|
||||
fixture.input.settle.readLifecycleState = () => ({
|
||||
aborted: true,
|
||||
timedOut: false,
|
||||
@@ -455,50 +435,6 @@ describe("finalizeEmbeddedAttemptStreamPhase", () => {
|
||||
expect(mocks.completeAfterTurn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("restores the rewound in-memory branch when prompt lock release fails", async () => {
|
||||
const sessionManager = SessionManager.inMemory();
|
||||
const promptId = sessionManager.appendMessage({
|
||||
role: "user",
|
||||
content: "Original request",
|
||||
timestamp: 1,
|
||||
});
|
||||
const rejectedId = sessionManager.appendMessage({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "Rejected first answer" }],
|
||||
stopReason: "stop",
|
||||
timestamp: 2,
|
||||
} as never);
|
||||
const originalMessages = sessionManager.buildSessionContext().messages;
|
||||
const activeSession = { agent: { state: { messages: originalMessages } } };
|
||||
const releaseError = new Error("prompt lock release failed");
|
||||
const fixture = createFixture({
|
||||
activeSession: activeSession as never,
|
||||
sessionManager: sessionManager as never,
|
||||
sessionLockController: {
|
||||
releaseForPrompt: vi.fn(async () => {
|
||||
throw releaseError;
|
||||
}),
|
||||
isPromptSubmissionBlockedError: () => false,
|
||||
} as never,
|
||||
repairedRejectedThinkingReplay: false,
|
||||
getBeforeAgentFinalizeRevisionEntryId: () => rejectedId,
|
||||
});
|
||||
|
||||
await expect(finalizeEmbeddedAttemptStreamPhase(fixture.input)).rejects.toBe(releaseError);
|
||||
|
||||
expect(sessionManager.getLeafId()).toBe(promptId);
|
||||
expect(activeSession.agent.state.messages).toEqual(
|
||||
sessionManager.buildSessionContext().messages,
|
||||
);
|
||||
expect(JSON.stringify(activeSession.agent.state.messages)).not.toContain(
|
||||
"Rejected first answer",
|
||||
);
|
||||
expect(mocks.settleStream).not.toHaveBeenCalled();
|
||||
expect(fixture.input.onSettleErrorState).not.toHaveBeenCalled();
|
||||
expect(fixture.input.onSettled).not.toHaveBeenCalled();
|
||||
expect(mocks.completeAfterTurn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("restores the rewound in-memory branch when settlement fails", async () => {
|
||||
const sessionManager = SessionManager.inMemory();
|
||||
const promptId = sessionManager.appendMessage({
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
/** Settles the provider stream and completes the post-turn lifecycle phase. */
|
||||
import { isRunnerAbortError } from "../abort.js";
|
||||
import { log } from "../logger.js";
|
||||
import { joinWithRunLivenessDeadline, RUN_LIVENESS_JOIN_TIMEOUT_MS } from "./abortable.js";
|
||||
import { completeEmbeddedAttemptAfterTurn } from "./attempt-after-turn.js";
|
||||
@@ -16,8 +15,7 @@ type SharedPhaseInputKeys =
|
||||
| "attempt"
|
||||
| "activeSession"
|
||||
| "sessionManager"
|
||||
| "sessionLockController"
|
||||
| "withOwnedSessionWriteLock";
|
||||
| "withOwnedTranscriptWrite";
|
||||
|
||||
// Queued subscription handlers (block-reply delivery, tool events) are
|
||||
// fire-and-forget during the turn; the pending-events join below is the only
|
||||
@@ -30,8 +28,7 @@ export async function finalizeEmbeddedAttemptStreamPhase(input: {
|
||||
attempt: StreamSettleInput["attempt"];
|
||||
activeSession: StreamSettleInput["activeSession"];
|
||||
sessionManager: StreamSettleInput["sessionManager"];
|
||||
sessionLockController: StreamSettleInput["sessionLockController"];
|
||||
withOwnedSessionWriteLock: StreamSettleInput["withOwnedSessionWriteLock"];
|
||||
withOwnedTranscriptWrite: StreamSettleInput["withOwnedTranscriptWrite"];
|
||||
waitForPendingEvents: () => Promise<void>;
|
||||
repairedRejectedThinkingReplay: boolean;
|
||||
getRunAbortDeadlineAtMs: () => number;
|
||||
@@ -51,7 +48,7 @@ export async function finalizeEmbeddedAttemptStreamPhase(input: {
|
||||
>;
|
||||
afterTurn: Omit<AfterTurnInput, SharedPhaseInputKeys | "state">;
|
||||
}): Promise<{ sessionIdUsed: string; sessionFileUsed?: string }> {
|
||||
const { activeSession, sessionManager, sessionLockController, withOwnedSessionWriteLock } = input;
|
||||
const { activeSession, sessionManager, withOwnedTranscriptWrite } = input;
|
||||
|
||||
await joinWithRunLivenessDeadline({
|
||||
joinWork: input.waitForPendingEvents,
|
||||
@@ -67,7 +64,7 @@ export async function finalizeEmbeddedAttemptStreamPhase(input: {
|
||||
const beforeAgentFinalizeRevisionEntryId = input.getBeforeAgentFinalizeRevisionEntryId();
|
||||
let rewoundBeforeAgentFinalizeRevision = false;
|
||||
if (beforeAgentFinalizeRevisionReason && beforeAgentFinalizeRevisionEntryId) {
|
||||
await withOwnedSessionWriteLock(() => {
|
||||
await withOwnedTranscriptWrite(() => {
|
||||
const rejectedEntry = sessionManager.getEntry(beforeAgentFinalizeRevisionEntryId);
|
||||
if (rejectedEntry?.type !== "message" || rejectedEntry.message.role !== "assistant") {
|
||||
throw new Error(
|
||||
@@ -89,20 +86,6 @@ export async function finalizeEmbeddedAttemptStreamPhase(input: {
|
||||
if (input.repairedRejectedThinkingReplay && !rewoundBeforeAgentFinalizeRevision) {
|
||||
activeSession.agent.state.messages = sessionManager.buildSessionContext().messages;
|
||||
}
|
||||
try {
|
||||
await sessionLockController.releaseForPrompt();
|
||||
} catch (err) {
|
||||
// An aborted attempt never submits another prompt, so a submission-blocked
|
||||
// release is expected teardown noise, including its recorded timeout reason.
|
||||
// Rethrowing would silently starve every agent_end consumer for aborted runs.
|
||||
const lifecycle = input.settle.readLifecycleState();
|
||||
const expectedAbortError =
|
||||
isRunnerAbortError(err) || sessionLockController.isPromptSubmissionBlockedError(err);
|
||||
if (!lifecycle.aborted || !expectedAbortError) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const currentState = input.getState();
|
||||
const streamSettleState = {
|
||||
promptError: currentState.promptError,
|
||||
@@ -115,8 +98,7 @@ export async function finalizeEmbeddedAttemptStreamPhase(input: {
|
||||
attempt: input.attempt,
|
||||
activeSession,
|
||||
sessionManager,
|
||||
sessionLockController,
|
||||
withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite,
|
||||
state: streamSettleState,
|
||||
...input.settle,
|
||||
runAbortDeadlineAtMs: input.getRunAbortDeadlineAtMs(),
|
||||
@@ -130,7 +112,7 @@ export async function finalizeEmbeddedAttemptStreamPhase(input: {
|
||||
}
|
||||
} finally {
|
||||
if (rewoundBeforeAgentFinalizeRevision) {
|
||||
await withOwnedSessionWriteLock(() => {
|
||||
await withOwnedTranscriptWrite(() => {
|
||||
// Settlement classifies the completed attempt from its original
|
||||
// in-memory messages. Later work always sees the rewound branch.
|
||||
activeSession.agent.state.messages = sessionManager.buildSessionContext().messages;
|
||||
@@ -146,8 +128,7 @@ export async function finalizeEmbeddedAttemptStreamPhase(input: {
|
||||
attempt: input.attempt,
|
||||
activeSession,
|
||||
sessionManager,
|
||||
sessionLockController,
|
||||
withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite,
|
||||
...input.afterTurn,
|
||||
state: {
|
||||
promptError: settledStream.promptError,
|
||||
|
||||
@@ -494,9 +494,6 @@ describe("prepareEmbeddedAttemptStream", () => {
|
||||
const markExternalAbort = vi.fn();
|
||||
const markAborted = vi.fn();
|
||||
const abortActiveSession = vi.fn(async () => {});
|
||||
const releaseHeldLockForAbort = vi.fn(
|
||||
async (_params: { reason?: unknown; terminal: boolean }) => {},
|
||||
);
|
||||
const abortState = {
|
||||
markAborted,
|
||||
markExternalAbort,
|
||||
@@ -527,7 +524,6 @@ describe("prepareEmbeddedAttemptStream", () => {
|
||||
isProbeSession: true,
|
||||
log: { warn: vi.fn() },
|
||||
runAbortController,
|
||||
sessionLockController: { releaseHeldLockForAbort },
|
||||
state: abortState,
|
||||
});
|
||||
externalAbortController.setRunAbort(abortRun);
|
||||
@@ -558,14 +554,7 @@ describe("prepareEmbeddedAttemptStream", () => {
|
||||
expect(onAttemptAbort).toHaveBeenCalledOnce();
|
||||
expect(markAborted).toHaveBeenCalledOnce();
|
||||
expect(abortActiveSession).toHaveBeenCalledOnce();
|
||||
expect(releaseHeldLockForAbort).toHaveBeenCalledOnce();
|
||||
expect(releaseHeldLockForAbort).toHaveBeenCalledWith({
|
||||
reason: expect.any(Error),
|
||||
terminal: true,
|
||||
});
|
||||
expect(
|
||||
isAgentRunSupersededAbortReason(releaseHeldLockForAbort.mock.calls[0]?.[0].reason),
|
||||
).toBe(true);
|
||||
expect(isAgentRunSupersededAbortReason(runAbortController.signal.reason)).toBe(true);
|
||||
expect(operation.result).toEqual({ kind: "failed", code: "run_stalled" });
|
||||
expect(operation.abortSignal.aborted).toBe(true);
|
||||
} finally {
|
||||
|
||||
@@ -33,7 +33,6 @@ type StreamGuardPhaseInput = Omit<
|
||||
| "onIdleTimeout"
|
||||
| "onRejectedThinkingReplayRepaired"
|
||||
| "session"
|
||||
| "sessionLockController"
|
||||
| "sessionManager"
|
||||
>;
|
||||
type HistoryPhaseInput = Omit<HistoryInput, "activeSession" | "attempt" | "sessionManager">;
|
||||
@@ -54,7 +53,6 @@ export async function prepareEmbeddedAttemptStreamRuntime(input: {
|
||||
activeSession: StreamInput["activeSession"];
|
||||
sessionManager: HistoryInput["sessionManager"] &
|
||||
NonNullable<ToolResultFlushInput["sessionManager"]>;
|
||||
sessionLockController: StreamGuardInput["sessionLockController"];
|
||||
ownedTranscriptWriteContext: Parameters<typeof withOwnedSessionTranscriptWrites>[0];
|
||||
runAbortController: AbortController;
|
||||
externalAbortController: ExternalAbortController;
|
||||
@@ -86,7 +84,6 @@ export async function prepareEmbeddedAttemptStreamRuntime(input: {
|
||||
attempt,
|
||||
session: activeSession,
|
||||
sessionManager,
|
||||
sessionLockController: input.sessionLockController,
|
||||
isYieldDetected: input.lifecycle.isYieldDetected,
|
||||
onRejectedThinkingReplayRepaired: input.lifecycle.markRejectedThinkingReplayRepaired,
|
||||
onIdleTimeout: (error) => idleTimeoutTriggerRef.current?.(error),
|
||||
@@ -123,7 +120,6 @@ export async function prepareEmbeddedAttemptStreamRuntime(input: {
|
||||
isProbeSession,
|
||||
log,
|
||||
runAbortController: input.runAbortController,
|
||||
sessionLockController: input.sessionLockController,
|
||||
state: input.abortState,
|
||||
});
|
||||
input.externalAbortController.setRunAbort(abortRun);
|
||||
|
||||
@@ -26,8 +26,7 @@ function createSettleFixture(overrides?: Partial<SettleInput>): SettleInput {
|
||||
messages: [],
|
||||
},
|
||||
sessionManager,
|
||||
sessionLockController: {},
|
||||
withOwnedSessionWriteLock: async (operation: () => unknown) => await operation(),
|
||||
withOwnedTranscriptWrite: async (operation: () => unknown) => await operation(),
|
||||
subscription: {
|
||||
toolMetas: [],
|
||||
waitForCompactionRetry: async () => {},
|
||||
|
||||
@@ -33,7 +33,6 @@ import {
|
||||
findLatestUncompactedAttemptUsageSnapshot,
|
||||
resolvePromptCacheTouchTimestamp,
|
||||
} from "./attempt.context-engine-helpers.js";
|
||||
import type { createEmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
import { appendAttemptCacheTtlIfNeeded } from "./attempt.thread-helpers.js";
|
||||
import {
|
||||
hasActiveCompactionRetryWork,
|
||||
@@ -43,14 +42,11 @@ import { selectCompactionTimeoutSnapshot } from "./compaction-timeout.js";
|
||||
import type { EmbeddedRunAttemptParams, EmbeddedRunAttemptResult } from "./types.js";
|
||||
|
||||
type EmbeddedAttemptSubscription = ReturnType<typeof subscribeEmbeddedAgentSession>;
|
||||
type AttemptSessionLockController = Awaited<
|
||||
ReturnType<typeof createEmbeddedAttemptSessionLockController>
|
||||
>;
|
||||
type PromptCacheRetention = Parameters<typeof buildContextEnginePromptCacheInfo>[0]["retention"];
|
||||
type ToolSearchTargetTranscriptProjections = Parameters<
|
||||
typeof projectToolSearchTargetTranscriptMessages
|
||||
>[1];
|
||||
type WithOwnedSessionWriteLock = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
type WithOwnedTranscriptWrite = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
|
||||
type StreamSettleResult = {
|
||||
promptError: unknown;
|
||||
@@ -72,8 +68,7 @@ export async function settleEmbeddedAttemptStream(input: {
|
||||
attempt: EmbeddedRunAttemptParams;
|
||||
activeSession: AgentSession;
|
||||
sessionManager: SessionManager;
|
||||
sessionLockController: AttemptSessionLockController;
|
||||
withOwnedSessionWriteLock: WithOwnedSessionWriteLock;
|
||||
withOwnedTranscriptWrite: WithOwnedTranscriptWrite;
|
||||
subscription: EmbeddedAttemptSubscription;
|
||||
state: {
|
||||
promptError: unknown;
|
||||
@@ -253,7 +248,7 @@ export async function settleEmbeddedAttemptStream(input: {
|
||||
let lastCallUsage: NormalizedUsage | undefined;
|
||||
let promptCache: EmbeddedRunAttemptResult["promptCache"];
|
||||
|
||||
await input.withOwnedSessionWriteLock(async () => {
|
||||
await input.withOwnedTranscriptWrite(async () => {
|
||||
const { timedOutDuringCompaction } = input.readLifecycleState();
|
||||
compactionOccurredThisAttempt = subscription.getCompactionCount() > 0;
|
||||
appendAttemptCacheTtlIfNeeded({
|
||||
|
||||
@@ -22,7 +22,6 @@ import {
|
||||
} from "../thinking.js";
|
||||
import { wrapStreamFnWithDiagnosticModelCallEvents } from "./attempt.model-diagnostic-events.js";
|
||||
import { resolveUnknownToolGuardThreshold } from "./attempt.run-decisions.js";
|
||||
import type { createEmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
import {
|
||||
createYieldAbortedResponse,
|
||||
isSessionsYieldAbortReason,
|
||||
@@ -51,10 +50,6 @@ import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type CacheTrace = ReturnType<typeof createCacheTrace>;
|
||||
type AnthropicPayloadLogger = ReturnType<typeof createAnthropicPayloadLogger>;
|
||||
type AttemptSessionLockController = Awaited<
|
||||
ReturnType<typeof createEmbeddedAttemptSessionLockController>
|
||||
>;
|
||||
|
||||
export function installEmbeddedAttemptStreamGuards(input: {
|
||||
attempt: EmbeddedRunAttemptParams;
|
||||
session: AgentSession;
|
||||
@@ -64,7 +59,6 @@ export function installEmbeddedAttemptStreamGuards(input: {
|
||||
systemPromptText: string;
|
||||
transcriptPolicy: TranscriptPolicy;
|
||||
sessionManager: SessionManager | undefined;
|
||||
sessionLockController: AttemptSessionLockController;
|
||||
isOpenAIResponsesApi: boolean;
|
||||
replayAllowedToolNames: Set<string>;
|
||||
liveAllowedToolNames: Set<string>;
|
||||
@@ -138,7 +132,6 @@ export function installEmbeddedAttemptStreamGuards(input: {
|
||||
});
|
||||
if (repair.repaired) {
|
||||
input.onRejectedThinkingReplayRepaired();
|
||||
input.sessionLockController.refreshAfterOwnedSessionWrite();
|
||||
return;
|
||||
}
|
||||
log.warn(
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createEmbeddedAttemptRunAbort } from "./attempt-abort.js";
|
||||
import { prepareEmbeddedAttemptTimeout } from "./attempt-timeout-prepare.js";
|
||||
import { createEmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
|
||||
function createTimeoutHarness(options?: { pendingCompaction?: boolean; timeoutMs?: number }) {
|
||||
const state = {
|
||||
@@ -67,26 +66,21 @@ describe("prepareEmbeddedAttemptTimeout", () => {
|
||||
harness.timeout.clearTimers();
|
||||
});
|
||||
|
||||
it("preserves the built-in deadline reason through a late prompt handoff", async () => {
|
||||
const sessionLockController = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
it("propagates the built-in deadline reason", async () => {
|
||||
const runAbortController = new AbortController();
|
||||
const abortRun = createEmbeddedAttemptRunAbort({
|
||||
abortActiveSession: vi.fn(async () => {}),
|
||||
activeSession: { abortCompaction: vi.fn(), isCompacting: false },
|
||||
attempt: {
|
||||
runId: "run-deadline-handoff",
|
||||
runId: "run-deadline",
|
||||
sessionFile: "agent:main:main",
|
||||
sessionId: "session-deadline-handoff",
|
||||
sessionId: "session-deadline",
|
||||
sessionKey: "agent:main:main",
|
||||
},
|
||||
getQueueHandle: () => undefined,
|
||||
isProbeSession: true,
|
||||
log: { warn: vi.fn() },
|
||||
runAbortController,
|
||||
sessionLockController,
|
||||
state: {
|
||||
markAborted: vi.fn(),
|
||||
markTimedOut: vi.fn(),
|
||||
@@ -96,8 +90,8 @@ describe("prepareEmbeddedAttemptTimeout", () => {
|
||||
});
|
||||
const timeout = prepareEmbeddedAttemptTimeout({
|
||||
attempt: {
|
||||
runId: "run-deadline-handoff",
|
||||
sessionId: "session-deadline-handoff",
|
||||
runId: "run-deadline",
|
||||
sessionId: "session-deadline",
|
||||
timeoutMs: 100,
|
||||
},
|
||||
activeSession: { isCompacting: false, isStreaming: false },
|
||||
@@ -111,13 +105,10 @@ describe("prepareEmbeddedAttemptTimeout", () => {
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
|
||||
const timeoutReason = runAbortController.signal.reason;
|
||||
expect(timeoutReason).toEqual(
|
||||
expect(runAbortController.signal.reason).toEqual(
|
||||
expect.objectContaining({ name: "TimeoutError", message: "request timed out" }),
|
||||
);
|
||||
await expect(sessionLockController.releaseForPrompt()).rejects.toBe(timeoutReason);
|
||||
timeout.clearTimers();
|
||||
await sessionLockController.dispose();
|
||||
});
|
||||
|
||||
it("grants one compaction grace window before aborting", async () => {
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { prepareEmbeddedAttemptTranscriptLifecycle } from "./attempt-transcript-lifecycle-prepare.js";
|
||||
|
||||
describe("prepareEmbeddedAttemptTranscriptLifecycle", () => {
|
||||
it("carries the admitted writer fence into nested transcript writes", async () => {
|
||||
const externalAbortController = {
|
||||
arm: vi.fn(),
|
||||
throwIfFiredAfterPrepCleanup: vi.fn(async () => undefined),
|
||||
};
|
||||
const prepared = await prepareEmbeddedAttemptTranscriptLifecycle({
|
||||
attempt: {
|
||||
config: {},
|
||||
runId: "run-a",
|
||||
sessionFile: "sqlite:session-a",
|
||||
sessionId: "session-a",
|
||||
sessionKey: "agent:main:test",
|
||||
sessionTarget: {
|
||||
agentId: "main",
|
||||
expectedLifecycleRevision: "revision-a",
|
||||
expectedWriterRunId: "run-a",
|
||||
sessionId: "session-a",
|
||||
sessionKey: "agent:main:test",
|
||||
storePath: "/tmp/openclaw.sqlite",
|
||||
},
|
||||
},
|
||||
externalAbortController,
|
||||
});
|
||||
|
||||
expect(prepared.ownedTranscriptWriteContext.sessionTarget).toMatchObject({
|
||||
expectedLifecycleRevision: "revision-a",
|
||||
expectedWriterRunId: "run-a",
|
||||
});
|
||||
await expect(prepared.withOwnedTranscriptWrite(async () => "done")).resolves.toBe("done");
|
||||
expect(externalAbortController.arm).toHaveBeenCalledOnce();
|
||||
expect(externalAbortController.throwIfFiredAfterPrepCleanup).toHaveBeenCalledTimes(2);
|
||||
await prepared.transcriptLifecycle.dispose();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
/** Prepares the admitted writer context and teardown tracker for one attempt. */
|
||||
import {
|
||||
type OwnedSessionTranscriptWriteContext,
|
||||
withOwnedSessionTranscriptWrites,
|
||||
} from "../../../config/sessions/transcript-write-context.js";
|
||||
import { resolveAgentRunSessionTarget } from "../../run-session-target.js";
|
||||
import { resolveCompactionTimeoutMs } from "../compaction-safety-timeout.js";
|
||||
import { createEmbeddedAttemptTranscriptLifecycle } from "./attempt-transcript-lifecycle.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
type WithOwnedTranscriptWrite = <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
|
||||
export async function prepareEmbeddedAttemptTranscriptLifecycle(input: {
|
||||
attempt: Pick<
|
||||
EmbeddedRunAttemptParams,
|
||||
| "abortSignal"
|
||||
| "config"
|
||||
| "runId"
|
||||
| "sessionFile"
|
||||
| "sessionId"
|
||||
| "sessionKey"
|
||||
| "sessionTarget"
|
||||
>;
|
||||
externalAbortController: {
|
||||
arm: () => void;
|
||||
throwIfFiredAfterPrepCleanup: () => Promise<void>;
|
||||
};
|
||||
}): Promise<{
|
||||
compactionTimeoutMs: number;
|
||||
ownedTranscriptWriteContext: OwnedSessionTranscriptWriteContext;
|
||||
transcriptLifecycle: ReturnType<typeof createEmbeddedAttemptTranscriptLifecycle>;
|
||||
withOwnedTranscriptWrite: WithOwnedTranscriptWrite;
|
||||
}> {
|
||||
const { attempt, externalAbortController } = input;
|
||||
const sessionTarget = await resolveAgentRunSessionTarget({
|
||||
agentId: attempt.sessionTarget?.agentId,
|
||||
config: attempt.config,
|
||||
missingSessionKey: "resolve-existing",
|
||||
sessionFile: attempt.sessionFile,
|
||||
sessionId: attempt.sessionId,
|
||||
sessionKey: attempt.sessionKey,
|
||||
sessionTarget: attempt.sessionTarget,
|
||||
});
|
||||
await externalAbortController.throwIfFiredAfterPrepCleanup();
|
||||
|
||||
const transcriptLifecycle = createEmbeddedAttemptTranscriptLifecycle({
|
||||
runId: attempt.runId,
|
||||
sessionId: attempt.sessionId,
|
||||
});
|
||||
const fencedSessionTarget = {
|
||||
...sessionTarget,
|
||||
...(attempt.sessionTarget?.expectedLifecycleRevision !== undefined
|
||||
? { expectedLifecycleRevision: attempt.sessionTarget.expectedLifecycleRevision }
|
||||
: {}),
|
||||
...(attempt.sessionTarget?.expectedWriterRunId !== undefined
|
||||
? { expectedWriterRunId: attempt.sessionTarget.expectedWriterRunId }
|
||||
: {}),
|
||||
};
|
||||
const ownedTranscriptWriteContext: OwnedSessionTranscriptWriteContext = {
|
||||
sessionFile: attempt.sessionFile,
|
||||
sessionKey: attempt.sessionKey,
|
||||
sessionTarget: fencedSessionTarget,
|
||||
withTranscriptWrite: (operation) => transcriptLifecycle.withTranscriptWrite(operation),
|
||||
};
|
||||
const withOwnedTranscriptWrite: WithOwnedTranscriptWrite = (operation) =>
|
||||
withOwnedSessionTranscriptWrites(ownedTranscriptWriteContext, async () =>
|
||||
transcriptLifecycle.withTranscriptWrite(operation),
|
||||
);
|
||||
|
||||
externalAbortController.arm();
|
||||
try {
|
||||
await externalAbortController.throwIfFiredAfterPrepCleanup();
|
||||
} catch (error) {
|
||||
await transcriptLifecycle.dispose();
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
compactionTimeoutMs: resolveCompactionTimeoutMs(attempt.config),
|
||||
ownedTranscriptWriteContext,
|
||||
transcriptLifecycle,
|
||||
withOwnedTranscriptWrite,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createEmbeddedAttemptTranscriptLifecycle } from "./attempt-transcript-lifecycle.js";
|
||||
|
||||
describe("createEmbeddedAttemptTranscriptLifecycle", () => {
|
||||
it("drains admitted transcript writes before cleanup continues", async () => {
|
||||
const lifecycle = createEmbeddedAttemptTranscriptLifecycle({
|
||||
runId: "run-a",
|
||||
sessionId: "session-a",
|
||||
});
|
||||
let releaseWrite = () => {};
|
||||
const writeGate = new Promise<void>((resolve) => {
|
||||
releaseWrite = resolve;
|
||||
});
|
||||
const write = lifecycle.withTranscriptWrite(async () => {
|
||||
await writeGate;
|
||||
});
|
||||
const cleanup = lifecycle.beginCleanup();
|
||||
|
||||
let cleanupSettled = false;
|
||||
void cleanup.then(() => {
|
||||
cleanupSettled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(cleanupSettled).toBe(false);
|
||||
|
||||
releaseWrite();
|
||||
await expect(write).resolves.toBeUndefined();
|
||||
await expect(cleanup).resolves.toBeUndefined();
|
||||
await expect(lifecycle.withTranscriptWrite(() => undefined)).rejects.toThrow(
|
||||
"attempt cleanup started before transcript write",
|
||||
);
|
||||
});
|
||||
|
||||
it("drains fire-and-forget nested writes before admitting the next writer", async () => {
|
||||
const lifecycle = createEmbeddedAttemptTranscriptLifecycle({});
|
||||
const order: string[] = [];
|
||||
await lifecycle.withTranscriptWrite(() => {
|
||||
order.push("outer");
|
||||
void lifecycle.withTranscriptWrite(async () => {
|
||||
await Promise.resolve();
|
||||
order.push("nested");
|
||||
});
|
||||
});
|
||||
await lifecycle.withTranscriptWrite(() => {
|
||||
order.push("next");
|
||||
});
|
||||
expect(order).toEqual(["outer", "nested", "next"]);
|
||||
});
|
||||
|
||||
it("rejects cleanup started from inside a transcript callback", async () => {
|
||||
const lifecycle = createEmbeddedAttemptTranscriptLifecycle({});
|
||||
await lifecycle.withTranscriptWrite(async () => {
|
||||
await expect(lifecycle.beginCleanup()).rejects.toThrow(
|
||||
"cannot start attempt cleanup inside a transcript write callback",
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,210 @@
|
||||
/** Serializes run-owned transcript callbacks and bounds teardown settlement. */
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import { toErrorObject } from "../../../infra/errors.js";
|
||||
import { log } from "../logger.js";
|
||||
|
||||
const TRANSCRIPT_TEARDOWN_BUDGET_MS = 30_000;
|
||||
|
||||
type LifecycleOwner = {
|
||||
active: boolean;
|
||||
nestedPending: number;
|
||||
nestedTail: Promise<void>;
|
||||
pendingOperations: Set<Promise<void>>;
|
||||
};
|
||||
|
||||
export type EmbeddedAttemptTranscriptLifecycle = {
|
||||
withTranscriptWrite<T>(run: () => Promise<T> | T): Promise<T>;
|
||||
beginCleanup(): Promise<void>;
|
||||
dispose(): Promise<void>;
|
||||
};
|
||||
|
||||
export function createEmbeddedAttemptTranscriptLifecycle(params: {
|
||||
runId?: string;
|
||||
sessionId?: string;
|
||||
}): EmbeddedAttemptTranscriptLifecycle {
|
||||
let cleanupRequested = false;
|
||||
let disposed = false;
|
||||
let lifecycle = Promise.resolve();
|
||||
let cleanupDrain: Promise<void> | undefined;
|
||||
let disposePromise: Promise<void> | undefined;
|
||||
let pendingWrites = 0;
|
||||
let teardownBudgetLogged = false;
|
||||
const lifecycleOwner = new AsyncLocalStorage<LifecycleOwner>();
|
||||
|
||||
const createLifecycleOwner = (): LifecycleOwner => ({
|
||||
active: true,
|
||||
nestedPending: 0,
|
||||
nestedTail: Promise.resolve(),
|
||||
pendingOperations: new Set(),
|
||||
});
|
||||
const drainLifecycleOwner = async (owner: LifecycleOwner): Promise<void> => {
|
||||
let firstError: Error | undefined;
|
||||
while (owner.pendingOperations.size > 0) {
|
||||
const pending = [...owner.pendingOperations];
|
||||
owner.pendingOperations.clear();
|
||||
const settled = await Promise.allSettled(pending);
|
||||
const rejected = settled.find((result) => result.status === "rejected");
|
||||
if (firstError === undefined && rejected?.status === "rejected") {
|
||||
firstError = toErrorObject(rejected.reason, "nested transcript write failed");
|
||||
}
|
||||
}
|
||||
if (firstError !== undefined) {
|
||||
throw firstError;
|
||||
}
|
||||
};
|
||||
const runLifecycleOwner = async <T>(
|
||||
owner: LifecycleOwner,
|
||||
run: () => Promise<T> | T,
|
||||
): Promise<T> => {
|
||||
let value: T | undefined;
|
||||
let primaryError: Error | undefined;
|
||||
try {
|
||||
value = await lifecycleOwner.run(owner, async () => await run());
|
||||
} catch (error) {
|
||||
primaryError = toErrorObject(error, "transcript write failed");
|
||||
}
|
||||
let drainError: Error | undefined;
|
||||
try {
|
||||
await drainLifecycleOwner(owner);
|
||||
} catch (error) {
|
||||
drainError = toErrorObject(error, "nested transcript drain failed");
|
||||
} finally {
|
||||
owner.active = false;
|
||||
}
|
||||
if (primaryError !== undefined) {
|
||||
if (
|
||||
drainError !== undefined &&
|
||||
drainError !== primaryError &&
|
||||
primaryError.cause === undefined
|
||||
) {
|
||||
try {
|
||||
primaryError.cause = drainError;
|
||||
} catch {
|
||||
// Frozen callback errors remain primary; drain failure is secondary.
|
||||
}
|
||||
}
|
||||
throw primaryError;
|
||||
}
|
||||
if (drainError !== undefined) {
|
||||
throw drainError;
|
||||
}
|
||||
return value as T;
|
||||
};
|
||||
const serializeLifecycle = async <T>(run: () => Promise<T> | T): Promise<T> => {
|
||||
const inheritedOwner = lifecycleOwner.getStore();
|
||||
if (inheritedOwner?.active) {
|
||||
const waitForPrevious =
|
||||
inheritedOwner.nestedPending > 0 ? inheritedOwner.nestedTail : Promise.resolve();
|
||||
inheritedOwner.nestedPending += 1;
|
||||
const operation = waitForPrevious.then(async () => {
|
||||
const childOwner = createLifecycleOwner();
|
||||
return await runLifecycleOwner(childOwner, run);
|
||||
});
|
||||
const queueTail = operation.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
const propagated = operation.then(() => undefined);
|
||||
void propagated.catch(() => {});
|
||||
inheritedOwner.nestedTail = queueTail;
|
||||
inheritedOwner.pendingOperations.add(propagated);
|
||||
void queueTail.finally(() => {
|
||||
inheritedOwner.nestedPending -= 1;
|
||||
});
|
||||
return await operation;
|
||||
}
|
||||
|
||||
const previous = lifecycle;
|
||||
let release!: () => void;
|
||||
lifecycle = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
await previous;
|
||||
const owner = createLifecycleOwner();
|
||||
try {
|
||||
return await runLifecycleOwner(owner, run);
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
};
|
||||
const logTeardownBudgetExpiry = (): void => {
|
||||
if (teardownBudgetLogged) {
|
||||
return;
|
||||
}
|
||||
teardownBudgetLogged = true;
|
||||
log.error(
|
||||
`transcript teardown budget expired: runId=${params.runId ?? "unknown"} ` +
|
||||
`sessionId=${params.sessionId ?? "unknown"} pendingWrites=${pendingWrites} ` +
|
||||
`timeoutMs=${TRANSCRIPT_TEARDOWN_BUDGET_MS}`,
|
||||
);
|
||||
};
|
||||
const settleWithinTeardownBudget = async (operation: Promise<void>): Promise<void> => {
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const settled = await Promise.race([
|
||||
operation.then(() => true),
|
||||
new Promise<false>((resolve) => {
|
||||
timeout = setTimeout(() => resolve(false), TRANSCRIPT_TEARDOWN_BUDGET_MS);
|
||||
timeout.unref?.();
|
||||
}),
|
||||
]);
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
if (!settled) {
|
||||
void operation.catch(() => {});
|
||||
logTeardownBudgetExpiry();
|
||||
}
|
||||
};
|
||||
const beginCleanup = async (): Promise<void> => {
|
||||
const currentOwner = lifecycleOwner.getStore();
|
||||
if (currentOwner?.active) {
|
||||
throw new Error("cannot start attempt cleanup inside a transcript write callback");
|
||||
}
|
||||
if (cleanupRequested) {
|
||||
if (cleanupDrain) {
|
||||
await cleanupDrain;
|
||||
}
|
||||
return;
|
||||
}
|
||||
cleanupRequested = true;
|
||||
cleanupDrain = settleWithinTeardownBudget(serializeLifecycle(() => {}));
|
||||
await cleanupDrain;
|
||||
};
|
||||
|
||||
return {
|
||||
withTranscriptWrite: (run) => {
|
||||
const activeDescendant = lifecycleOwner.getStore()?.active === true;
|
||||
if ((cleanupRequested || disposed) && !activeDescendant) {
|
||||
const rejected = Promise.reject(
|
||||
new Error(
|
||||
disposed
|
||||
? "attempt disposed before transcript write"
|
||||
: "attempt cleanup started before transcript write",
|
||||
),
|
||||
);
|
||||
void rejected.catch(() => {});
|
||||
return rejected;
|
||||
}
|
||||
pendingWrites += 1;
|
||||
const operation = serializeLifecycle(run);
|
||||
void operation
|
||||
.finally(() => {
|
||||
pendingWrites -= 1;
|
||||
})
|
||||
.catch(() => {});
|
||||
return operation;
|
||||
},
|
||||
beginCleanup,
|
||||
dispose: async () => {
|
||||
const currentOwner = lifecycleOwner.getStore();
|
||||
if (currentOwner?.active) {
|
||||
throw new Error("cannot dispose an attempt from inside a transcript write callback");
|
||||
}
|
||||
disposePromise ??= (async () => {
|
||||
await beginCleanup();
|
||||
disposed = true;
|
||||
})();
|
||||
await disposePromise;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { buildAgentRunTerminalOutcomeFromAttempt } from "../../agent-run-terminal-outcome.js";
|
||||
import {
|
||||
cleanupTempPaths,
|
||||
@@ -26,44 +26,6 @@ describe("runEmbeddedAttempt abort races", () => {
|
||||
tempPaths.length = 0;
|
||||
});
|
||||
|
||||
it("stops before session creation when aborted during eager lock acquisition", async () => {
|
||||
const abortController = new AbortController();
|
||||
const prompt = vi.fn(async () => {});
|
||||
const abortError = new Error("stopped during lock acquisition");
|
||||
abortError.name = "AbortError";
|
||||
let markLockRequested!: () => void;
|
||||
let observedSignal: AbortSignal | undefined;
|
||||
const lockRequested = new Promise<void>((resolve) => {
|
||||
markLockRequested = resolve;
|
||||
});
|
||||
hoisted.acquireSessionWriteLockMock.mockImplementationOnce(async (params) => {
|
||||
observedSignal = params.signal;
|
||||
markLockRequested();
|
||||
await new Promise<void>((resolve) => {
|
||||
params.signal?.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
throw params.signal?.reason;
|
||||
});
|
||||
|
||||
const attempt = createContextEngineAttemptRunner({
|
||||
contextEngine: createContextEngineBootstrapAndAssemble(),
|
||||
sessionKey: "agent:main:telegram:direct:123",
|
||||
tempPaths,
|
||||
sessionPrompt: prompt,
|
||||
attemptOverrides: {
|
||||
abortSignal: abortController.signal,
|
||||
},
|
||||
});
|
||||
await lockRequested;
|
||||
abortController.abort(abortError);
|
||||
|
||||
await expect(attempt).rejects.toBe(abortError);
|
||||
|
||||
expect(hoisted.createAgentSessionMock).not.toHaveBeenCalled();
|
||||
expect(prompt).not.toHaveBeenCalled();
|
||||
expect(observedSignal).toBe(abortController.signal);
|
||||
});
|
||||
|
||||
it("preserves a run-budget timeout when abort blocks prompt submission", async () => {
|
||||
let releasePendingEvents!: () => void;
|
||||
const pendingEvents = new Promise<void>((resolve) => {
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
// Coverage for small run-attempt decision helpers.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveCredentialScopedAuthAttemptModelDecision } from "../../runtime-plan/credential-scoped-model.js";
|
||||
import {
|
||||
resolveAttemptStreamAuthProfileId,
|
||||
resolveAttemptToolPolicyMessageProvider,
|
||||
resolveEmbeddedAttemptSessionWriteLockOptions,
|
||||
resolveUnknownToolGuardThreshold,
|
||||
shouldRunLlmOutputHooksForAttempt,
|
||||
} from "./attempt.run-decisions.js";
|
||||
|
||||
describe("resolveEmbeddedAttemptSessionWriteLockOptions", () => {
|
||||
it("derives the dead-process lease TTL from the compaction timeout", () => {
|
||||
// Live owners renew until explicit release; this only bounds stale metadata after owner death.
|
||||
const options = resolveEmbeddedAttemptSessionWriteLockOptions({
|
||||
config: {},
|
||||
compactionTimeoutMs: 600_000,
|
||||
env: {},
|
||||
});
|
||||
|
||||
expect(options.maxHoldMs).toBe(720_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveAttemptStreamAuthProfileId", () => {
|
||||
it("uses only the runtime-forwarded auth profile for stream provenance", () => {
|
||||
// Raw attempt authProfileId may be a session selection detail; stream
|
||||
// provenance should only expose the runtime-forwarded profile.
|
||||
expect(
|
||||
resolveAttemptStreamAuthProfileId({
|
||||
authProfileId: "openai:raw-session-profile",
|
||||
runtimePlan: {
|
||||
auth: {
|
||||
forwardedAuthProfileId: "openai:forwarded-profile",
|
||||
},
|
||||
} as never,
|
||||
}),
|
||||
).toBe("openai:forwarded-profile");
|
||||
|
||||
expect(
|
||||
resolveAttemptStreamAuthProfileId({
|
||||
authProfileId: "openai:non-forwarded-profile",
|
||||
runtimePlan: {
|
||||
auth: {},
|
||||
} as never,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
describe("resolveCredentialScopedAuthAttemptModelDecision", () => {
|
||||
const resolveDecision = (
|
||||
plan: Record<string, unknown>,
|
||||
requestedProfileId?: string,
|
||||
providerUsesProfileScopedModelMetadata = false,
|
||||
) =>
|
||||
resolveCredentialScopedAuthAttemptModelDecision({
|
||||
attempt: { kind: "implicit", plan } as never,
|
||||
priorProfileAttempted: false,
|
||||
requestedProfileId,
|
||||
providerUsesProfileScopedModelMetadata,
|
||||
});
|
||||
|
||||
it("materializes route-less plans when a provider profile scopes model metadata", () => {
|
||||
expect(resolveDecision({ forwardedAuthProfileId: "github-copilot:work" })).toMatchObject({
|
||||
forceResolve: false,
|
||||
shouldMaterialize: true,
|
||||
});
|
||||
expect(resolveDecision({}, "github-copilot:requested").shouldMaterialize).toBe(true);
|
||||
expect(resolveDecision({ selectedAuthMode: "api-key" }, undefined, true)).toMatchObject({
|
||||
forceResolve: false,
|
||||
shouldMaterialize: true,
|
||||
authRequirement: "api-key",
|
||||
});
|
||||
expect(
|
||||
resolveDecision({ selectedAuthMode: "api-key" }, undefined, false).shouldMaterialize,
|
||||
).toBe(false);
|
||||
expect(resolveDecision({}).shouldMaterialize).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveAttemptToolPolicyMessageProvider", () => {
|
||||
it("prefers explicit tool-policy provider over transport channel", () => {
|
||||
expect(
|
||||
resolveAttemptToolPolicyMessageProvider({
|
||||
messageChannel: "discord",
|
||||
messageProvider: "discord-voice",
|
||||
}),
|
||||
).toBe("discord-voice");
|
||||
});
|
||||
|
||||
it("falls back to message channel when provider is omitted", () => {
|
||||
expect(resolveAttemptToolPolicyMessageProvider({ messageChannel: "discord" })).toBe("discord");
|
||||
});
|
||||
});
|
||||
|
||||
describe("shouldRunLlmOutputHooksForAttempt", () => {
|
||||
it("skips llm_output after before_agent_run blocks before model submission", () => {
|
||||
expect(shouldRunLlmOutputHooksForAttempt({ promptErrorSource: "hook:before_agent_run" })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(shouldRunLlmOutputHooksForAttempt({ promptErrorSource: "prompt" })).toBe(true);
|
||||
expect(shouldRunLlmOutputHooksForAttempt({ promptErrorSource: null })).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveUnknownToolGuardThreshold", () => {
|
||||
it("returns the default threshold when no loop-detection config is provided", () => {
|
||||
expect(resolveUnknownToolGuardThreshold(undefined)).toBe(10);
|
||||
expect(resolveUnknownToolGuardThreshold({})).toBe(10);
|
||||
});
|
||||
|
||||
it("stays on even when tools.loopDetection.enabled is false", () => {
|
||||
// Unknown-tool guard is a model-safety circuit, separate from configurable
|
||||
// repeated-tool loop detection.
|
||||
expect(resolveUnknownToolGuardThreshold({ enabled: false })).toBe(10);
|
||||
});
|
||||
});
|
||||
@@ -1,30 +1,9 @@
|
||||
/**
|
||||
* Resolves per-attempt runtime decisions from config and channel context.
|
||||
*/
|
||||
import type { OpenClawConfig } from "../../../config/config.js";
|
||||
import {
|
||||
resolveSessionLockMaxHoldFromTimeout,
|
||||
resolveSessionWriteLockOptions,
|
||||
} from "../../session-write-lock.js";
|
||||
import { UNKNOWN_TOOL_THRESHOLD } from "../../tool-loop-detection.js";
|
||||
import type { EmbeddedRunAttemptParams } from "./types.js";
|
||||
|
||||
/** Builds the session write-lock timing for a live embedded attempt. */
|
||||
export function resolveEmbeddedAttemptSessionWriteLockOptions(params: {
|
||||
config?: OpenClawConfig;
|
||||
compactionTimeoutMs: number;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): { timeoutMs: number; staleMs: number; maxHoldMs: number } {
|
||||
// maxHoldMs is the SQLite lease TTL for dead-process reclamation, not a live-owner watchdog.
|
||||
// A live process renews the lease until teardown releases it.
|
||||
return resolveSessionWriteLockOptions(params.config, {
|
||||
env: params.env,
|
||||
maxHoldMsFallback: resolveSessionLockMaxHoldFromTimeout({
|
||||
timeoutMs: params.compactionTimeoutMs,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the auth profile id that should be attached to model-stream
|
||||
* provenance. Only runtime-forwarded ids are exposed; raw request auth ids can
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
loadTranscriptEvents,
|
||||
upsertSessionEntry,
|
||||
} from "../../../config/sessions/session-accessor.js";
|
||||
import { withOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js";
|
||||
import { withOpenClawTestState } from "../../../test-utils/openclaw-test-state.js";
|
||||
import { SessionWriteLockStaleError } from "../../session-write-lock-error.js";
|
||||
import { SessionManager } from "../../sessions/session-manager.js";
|
||||
import { createEmbeddedAttemptSessionLockController } from "./attempt.session-lock.js";
|
||||
|
||||
describe("attempt session-lock teardown budgets", () => {
|
||||
it("bounds cleanup acquisition and releases before a started writer settles", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let markWriteStarted!: () => void;
|
||||
const writeStarted = new Promise<void>((resolve) => {
|
||||
markWriteStarted = resolve;
|
||||
});
|
||||
let resumeWrite!: () => void;
|
||||
const writeBlocked = new Promise<void>((resolve) => {
|
||||
resumeWrite = resolve;
|
||||
});
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
runId: "cleanup-budget-run",
|
||||
sessionId: "cleanup-budget-session",
|
||||
});
|
||||
const write = controller.withSessionWriteLock(async () => {
|
||||
markWriteStarted();
|
||||
await writeBlocked;
|
||||
});
|
||||
await writeStarted;
|
||||
|
||||
const cleanupAcquisition = controller.acquireForCleanup();
|
||||
await vi.advanceTimersByTimeAsync(29_999);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
const cleanupLock = await cleanupAcquisition;
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
expect(controller.hasSessionTakeover()).toBe(true);
|
||||
|
||||
await cleanupLock.release();
|
||||
await expect(controller.dispose()).resolves.toBeUndefined();
|
||||
resumeWrite();
|
||||
await expect(write).resolves.toBeUndefined();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("force-releases a started writer at the teardown budget and fences its late persist", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
await withOpenClawTestState({ label: "attempt-session-lock-late-persist" }, async (state) => {
|
||||
const target = {
|
||||
agentId: "main",
|
||||
sessionId: "late-persist-session",
|
||||
sessionKey: "agent:main:late-persist-session",
|
||||
storePath: state.path("sessions.json"),
|
||||
};
|
||||
await upsertSessionEntry(target, { sessionId: target.sessionId, updatedAt: 1 });
|
||||
const sessionManager = SessionManager.open(target, state.workspaceDir);
|
||||
sessionManager.appendModelChange("test", "before");
|
||||
const before = await loadTranscriptEvents(target);
|
||||
const leaseError = new SessionWriteLockStaleError({
|
||||
lockPath: `sqlite:session-write:${target.sessionKey}`,
|
||||
owner: "released teardown lease",
|
||||
staleReasons: ["lease-lost"],
|
||||
});
|
||||
let owned = true;
|
||||
const release = vi.fn(async () => {
|
||||
owned = false;
|
||||
});
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({
|
||||
assertOwned: () => {
|
||||
if (!owned) {
|
||||
throw leaseError;
|
||||
}
|
||||
},
|
||||
release,
|
||||
})),
|
||||
lockOptions: { sessionFile: target.sessionKey },
|
||||
runId: "late-persist-run",
|
||||
sessionId: target.sessionId,
|
||||
});
|
||||
let markWriteStarted!: () => void;
|
||||
const writeStarted = new Promise<void>((resolve) => {
|
||||
markWriteStarted = resolve;
|
||||
});
|
||||
let resumeWrite!: () => void;
|
||||
const writeBlocked = new Promise<void>((resolve) => {
|
||||
resumeWrite = resolve;
|
||||
});
|
||||
const write = withOwnedSessionTranscriptWrites(
|
||||
{
|
||||
sessionFile: target.sessionKey,
|
||||
sessionKey: target.sessionKey,
|
||||
sessionTarget: target,
|
||||
assertOwned: () => controller.assertOwned(),
|
||||
withSessionWriteLock: (run, options) => controller.withSessionWriteLock(run, options),
|
||||
},
|
||||
async () =>
|
||||
await controller.withSessionWriteLock(async () => {
|
||||
markWriteStarted();
|
||||
await writeBlocked;
|
||||
sessionManager.appendModelChange("test", "too-late");
|
||||
}),
|
||||
);
|
||||
await writeStarted;
|
||||
|
||||
const disposal = controller.dispose();
|
||||
await vi.advanceTimersByTimeAsync(5_000 + 29_999);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await expect(disposal).resolves.toBeUndefined();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
|
||||
resumeWrite();
|
||||
await expect(write).rejects.toBe(leaseError);
|
||||
await expect(loadTranscriptEvents(target)).resolves.toEqual(before);
|
||||
});
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("lets a started persist settle within the teardown budget before releasing", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ label: "attempt-session-lock-settled-persist" },
|
||||
async (state) => {
|
||||
const target = {
|
||||
agentId: "main",
|
||||
sessionId: "settled-persist-session",
|
||||
sessionKey: "agent:main:settled-persist-session",
|
||||
storePath: state.path("sessions.json"),
|
||||
};
|
||||
await upsertSessionEntry(target, { sessionId: target.sessionId, updatedAt: 1 });
|
||||
const sessionManager = SessionManager.open(target, state.workspaceDir);
|
||||
sessionManager.appendModelChange("test", "before");
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ assertOwned: () => undefined, release })),
|
||||
lockOptions: { sessionFile: target.sessionKey },
|
||||
});
|
||||
let markWriteStarted!: () => void;
|
||||
const writeStarted = new Promise<void>((resolve) => {
|
||||
markWriteStarted = resolve;
|
||||
});
|
||||
let resumeWrite!: () => void;
|
||||
const writeBlocked = new Promise<void>((resolve) => {
|
||||
resumeWrite = resolve;
|
||||
});
|
||||
const write = withOwnedSessionTranscriptWrites(
|
||||
{
|
||||
sessionFile: target.sessionKey,
|
||||
sessionKey: target.sessionKey,
|
||||
sessionTarget: target,
|
||||
assertOwned: () => controller.assertOwned(),
|
||||
withSessionWriteLock: (run, options) => controller.withSessionWriteLock(run, options),
|
||||
},
|
||||
async () =>
|
||||
await controller.withSessionWriteLock(async () => {
|
||||
markWriteStarted();
|
||||
await writeBlocked;
|
||||
sessionManager.appendModelChange("test", "within-budget");
|
||||
}),
|
||||
);
|
||||
await writeStarted;
|
||||
const disposal = controller.dispose();
|
||||
await Promise.resolve();
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
|
||||
resumeWrite();
|
||||
await Promise.all([write, disposal]);
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
await expect(loadTranscriptEvents(target)).resolves.toContainEqual(
|
||||
expect.objectContaining({ modelId: "within-budget", type: "model_change" }),
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,936 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionWriteLockStaleError } from "../../session-write-lock-error.js";
|
||||
import {
|
||||
createEmbeddedAttemptSessionLockController,
|
||||
EmbeddedAttemptSessionTakeoverError,
|
||||
installPromptSubmissionLockRelease,
|
||||
} from "./attempt.session-lock.js";
|
||||
|
||||
describe("createEmbeddedAttemptSessionLockController", () => {
|
||||
it("reloads SQLite transcript state after prompt-time writers finish", async () => {
|
||||
const reloadPromptReleasedSessionFile = vi.fn(async () => undefined);
|
||||
const acquireSessionWriteLock = vi.fn(async () => ({ release: async () => undefined }));
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock,
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile,
|
||||
});
|
||||
|
||||
await controller.reacquireAfterPrompt();
|
||||
|
||||
expect(reloadPromptReleasedSessionFile).toHaveBeenCalledOnce();
|
||||
expect(acquireSessionWriteLock).toHaveBeenCalledWith({
|
||||
sessionFile: "agent:main:main",
|
||||
targetKind: "session-key",
|
||||
});
|
||||
});
|
||||
|
||||
it("serializes the complete SQLite write callbacks", async () => {
|
||||
let releaseFirst!: () => void;
|
||||
const firstBlocked = new Promise<void>((resolve) => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
const events: string[] = [];
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
const first = controller.withSessionWriteLock(async () => {
|
||||
events.push("first:start");
|
||||
await firstBlocked;
|
||||
events.push("first:end");
|
||||
});
|
||||
const second = controller.withSessionWriteLock(() => {
|
||||
events.push("second");
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(events).toEqual(["first:start"]));
|
||||
releaseFirst();
|
||||
await Promise.all([first, second]);
|
||||
|
||||
expect(events).toEqual(["first:start", "first:end", "second"]);
|
||||
});
|
||||
|
||||
it("fences transcript writes after the SQLite session lease is lost", async () => {
|
||||
const leaseError = new SessionWriteLockStaleError({
|
||||
lockPath: "sqlite:session-write:agent:main:main",
|
||||
owner: "replacement",
|
||||
staleReasons: ["lease-lost"],
|
||||
});
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({
|
||||
assertOwned: () => {
|
||||
throw leaseError;
|
||||
},
|
||||
release: async () => undefined,
|
||||
})),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await expect(controller.withSessionWriteLock(() => undefined)).rejects.toBe(leaseError);
|
||||
expect(controller.hasSessionTakeover()).toBe(true);
|
||||
});
|
||||
|
||||
it("blocks prompt submission after the SQLite session lease is lost", async () => {
|
||||
const leaseError = new SessionWriteLockStaleError({
|
||||
lockPath: "sqlite:session-write:agent:main:main",
|
||||
owner: "replacement",
|
||||
staleReasons: ["lease-lost"],
|
||||
});
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({
|
||||
assertOwned: () => {
|
||||
throw leaseError;
|
||||
},
|
||||
release: async () => undefined,
|
||||
})),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await expect(controller.releaseForPrompt()).rejects.toBe(leaseError);
|
||||
expect(controller.hasSessionTakeover()).toBe(true);
|
||||
});
|
||||
|
||||
it("allows nested writes to reuse the active lifecycle owner", async () => {
|
||||
const events: string[] = [];
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await controller.withSessionWriteLock(async () => {
|
||||
events.push("outer:start");
|
||||
await controller.withSessionWriteLock(() => {
|
||||
events.push("nested");
|
||||
});
|
||||
events.push("outer:end");
|
||||
});
|
||||
|
||||
expect(events).toEqual(["outer:start", "nested", "outer:end"]);
|
||||
});
|
||||
|
||||
it("queues async descendants that resume after their lifecycle owner exits", async () => {
|
||||
let resumeDescendant!: () => void;
|
||||
const descendantBlocked = new Promise<void>((resolve) => {
|
||||
resumeDescendant = resolve;
|
||||
});
|
||||
let releaseSecond!: () => void;
|
||||
const secondBlocked = new Promise<void>((resolve) => {
|
||||
releaseSecond = resolve;
|
||||
});
|
||||
const events: string[] = [];
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
let descendant!: Promise<void>;
|
||||
|
||||
await controller.withSessionWriteLock(() => {
|
||||
descendant = (async () => {
|
||||
await descendantBlocked;
|
||||
await controller.withSessionWriteLock(() => {
|
||||
events.push("descendant");
|
||||
});
|
||||
})();
|
||||
});
|
||||
const second = controller.withSessionWriteLock(async () => {
|
||||
events.push("second:start");
|
||||
await secondBlocked;
|
||||
events.push("second:end");
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(events).toEqual(["second:start"]));
|
||||
resumeDescendant();
|
||||
await Promise.resolve();
|
||||
expect(events).toEqual(["second:start"]);
|
||||
releaseSecond();
|
||||
await Promise.all([second, descendant]);
|
||||
|
||||
expect(events).toEqual(["second:start", "second:end", "descendant"]);
|
||||
});
|
||||
|
||||
it("keeps the lifecycle held until detached nested writes settle", async () => {
|
||||
let releaseNested!: () => void;
|
||||
const nestedBlocked = new Promise<void>((resolve) => {
|
||||
releaseNested = resolve;
|
||||
});
|
||||
const events: string[] = [];
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
const outer = controller.withSessionWriteLock(() => {
|
||||
void controller.withSessionWriteLock(async () => {
|
||||
events.push("nested:start");
|
||||
await nestedBlocked;
|
||||
events.push("nested:end");
|
||||
});
|
||||
});
|
||||
const second = controller.withSessionWriteLock(() => {
|
||||
events.push("second");
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(events).toEqual(["nested:start"]));
|
||||
releaseNested();
|
||||
await Promise.all([outer, second]);
|
||||
|
||||
expect(events).toEqual(["nested:start", "nested:end", "second"]);
|
||||
});
|
||||
|
||||
it("serializes sibling nested writes in submission order", async () => {
|
||||
let releaseFirst!: () => void;
|
||||
const firstBlocked = new Promise<void>((resolve) => {
|
||||
releaseFirst = resolve;
|
||||
});
|
||||
const events: string[] = [];
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await controller.withSessionWriteLock(async () => {
|
||||
const first = controller.withSessionWriteLock(async () => {
|
||||
events.push("first:start");
|
||||
await firstBlocked;
|
||||
events.push("first:end");
|
||||
});
|
||||
const second = controller.withSessionWriteLock(() => {
|
||||
events.push("second");
|
||||
});
|
||||
await vi.waitFor(() => expect(events).toEqual(["first:start"]));
|
||||
releaseFirst();
|
||||
await Promise.all([first, second]);
|
||||
});
|
||||
|
||||
expect(events).toEqual(["first:start", "first:end", "second"]);
|
||||
});
|
||||
|
||||
it("propagates failures from detached nested writes", async () => {
|
||||
const nestedError = new Error("nested write failed");
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await expect(
|
||||
controller.withSessionWriteLock(() => {
|
||||
void controller.withSessionWriteLock(() => {
|
||||
throw nestedError;
|
||||
});
|
||||
}),
|
||||
).rejects.toBe(nestedError);
|
||||
});
|
||||
|
||||
it("waits for every detached nested write before propagating failure", async () => {
|
||||
const nestedError = new Error("nested write failed");
|
||||
let releaseSlowWrite!: () => void;
|
||||
const slowWriteBlocked = new Promise<void>((resolve) => {
|
||||
releaseSlowWrite = resolve;
|
||||
});
|
||||
const slowWriteStarted = vi.fn();
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
const completion = controller.withSessionWriteLock(() => {
|
||||
void controller.withSessionWriteLock(() => {
|
||||
throw nestedError;
|
||||
});
|
||||
void controller.withSessionWriteLock(async () => {
|
||||
slowWriteStarted();
|
||||
await slowWriteBlocked;
|
||||
});
|
||||
});
|
||||
let settled = false;
|
||||
void completion.then(
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
);
|
||||
await vi.waitFor(() => expect(slowWriteStarted).toHaveBeenCalledOnce());
|
||||
|
||||
expect(settled).toBe(false);
|
||||
releaseSlowWrite();
|
||||
await expect(completion).rejects.toBe(nestedError);
|
||||
});
|
||||
|
||||
it("preserves the callback error when a handled nested write also failed", async () => {
|
||||
const primaryError = new Error("outer write failed");
|
||||
const nestedError = new Error("nested write failed");
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await expect(
|
||||
controller.withSessionWriteLock(async () => {
|
||||
await controller
|
||||
.withSessionWriteLock(() => {
|
||||
throw nestedError;
|
||||
})
|
||||
.catch(() => undefined);
|
||||
throw primaryError;
|
||||
}),
|
||||
).rejects.toBe(primaryError);
|
||||
expect(primaryError.cause).toBe(nestedError);
|
||||
});
|
||||
|
||||
it("preserves a frozen callback error when nested drain also failed", async () => {
|
||||
const primaryError = new Error("outer write failed");
|
||||
Object.freeze(primaryError);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await expect(
|
||||
controller.withSessionWriteLock(async () => {
|
||||
await controller
|
||||
.withSessionWriteLock(() => {
|
||||
throw new Error("nested write failed");
|
||||
})
|
||||
.catch(() => undefined);
|
||||
throw primaryError;
|
||||
}),
|
||||
).rejects.toBe(primaryError);
|
||||
});
|
||||
|
||||
it("does not attach a propagated nested error as its own cause", async () => {
|
||||
const nestedError = new Error("nested write failed");
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await expect(
|
||||
controller.withSessionWriteLock(async () => {
|
||||
await controller.withSessionWriteLock(() => {
|
||||
throw nestedError;
|
||||
});
|
||||
}),
|
||||
).rejects.toBe(nestedError);
|
||||
expect(nestedError.cause).toBeUndefined();
|
||||
});
|
||||
|
||||
it("terminally fences writes after a generic prompt reload failure", async () => {
|
||||
const reloadError = new Error("reload failed");
|
||||
const run = vi.fn();
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile: () => {
|
||||
throw reloadError;
|
||||
},
|
||||
});
|
||||
|
||||
await expect(controller.reacquireAfterPrompt()).rejects.toBe(reloadError);
|
||||
await expect(controller.withSessionWriteLock(run)).rejects.toBe(reloadError);
|
||||
await controller.releaseForPrompt();
|
||||
await expect(controller.reacquireAfterPrompt()).rejects.toBe(reloadError);
|
||||
await expect(controller.acquireForCleanup()).resolves.toBeDefined();
|
||||
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
expect(controller.hasSessionTakeover()).toBe(false);
|
||||
});
|
||||
|
||||
it("marks only takeover reload failures as session takeover", async () => {
|
||||
const takeoverError = new EmbeddedAttemptSessionTakeoverError("agent:main:main");
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile: () => {
|
||||
throw takeoverError;
|
||||
},
|
||||
});
|
||||
|
||||
await expect(controller.reacquireAfterPrompt()).rejects.toBe(takeoverError);
|
||||
|
||||
expect(controller.hasSessionTakeover()).toBe(true);
|
||||
});
|
||||
|
||||
it("does not wait on a stalled reload after the disposal timeout", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile: async () => await new Promise<void>(() => {}),
|
||||
});
|
||||
|
||||
await controller.releaseForPrompt();
|
||||
void controller.reacquireAfterPrompt();
|
||||
await Promise.resolve();
|
||||
const disposal = controller.dispose();
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
|
||||
await expect(disposal).resolves.toBeUndefined();
|
||||
await expect(controller.releaseForPrompt()).rejects.toThrow(
|
||||
"attempt disposed before prompt submission",
|
||||
);
|
||||
await expect(controller.reacquireAfterPrompt()).resolves.toBeUndefined();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps queued writes inside the disposal timeout when prompt reload stalls", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let markReloadStarted!: () => void;
|
||||
const reloadStarted = new Promise<void>((resolve) => {
|
||||
markReloadStarted = resolve;
|
||||
});
|
||||
let finishReload!: () => void;
|
||||
const reloadBlocked = new Promise<void>((resolve) => {
|
||||
finishReload = resolve;
|
||||
});
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile: async () => {
|
||||
markReloadStarted();
|
||||
await reloadBlocked;
|
||||
},
|
||||
});
|
||||
await controller.releaseForPrompt();
|
||||
const reacquire = controller.reacquireAfterPrompt();
|
||||
await reloadStarted;
|
||||
const writeCallback = vi.fn();
|
||||
const queuedWrite = controller.withSessionWriteLock(writeCallback);
|
||||
const queuedWriteOutcome = queuedWrite.then(
|
||||
() => undefined,
|
||||
(error: unknown) => error,
|
||||
);
|
||||
|
||||
const disposal = controller.dispose();
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await expect(disposal).resolves.toBeUndefined();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
expect(writeCallback).not.toHaveBeenCalled();
|
||||
|
||||
finishReload();
|
||||
await expect(reacquire).resolves.toBeUndefined();
|
||||
await expect(queuedWriteOutcome).resolves.toEqual(
|
||||
expect.objectContaining({ message: "attempt disposed before transcript write" }),
|
||||
);
|
||||
expect(writeCallback).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects disposal from inside a write callback", async () => {
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await controller.withSessionWriteLock(async () => {
|
||||
await expect(controller.dispose()).rejects.toThrow(
|
||||
"cannot dispose an attempt from inside a transcript write callback",
|
||||
);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
});
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
await controller.dispose();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("rejects cleanup from inside a write callback", async () => {
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await controller.withSessionWriteLock(async () => {
|
||||
await expect(controller.acquireForCleanup()).rejects.toThrow(
|
||||
"cannot start attempt cleanup inside a transcript write callback",
|
||||
);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
});
|
||||
const cleanupLock = await controller.acquireForCleanup();
|
||||
await expect(controller.withSessionWriteLock(async () => undefined)).rejects.toThrow(
|
||||
"attempt cleanup started before transcript write",
|
||||
);
|
||||
await cleanupLock.release();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps cleanup ownership until the granted cleanup lock is released", async () => {
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
const cleanupLock = await controller.acquireForCleanup();
|
||||
|
||||
let disposeSettled = false;
|
||||
const disposal = controller.dispose().then(() => {
|
||||
disposeSettled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
|
||||
expect(disposeSettled).toBe(false);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
await cleanupLock.release();
|
||||
await disposal;
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("retries the underlying release when cleanup release fails", async () => {
|
||||
const releaseError = new Error("release failed");
|
||||
const release = vi
|
||||
.fn<() => Promise<void>>()
|
||||
.mockRejectedValueOnce(releaseError)
|
||||
.mockResolvedValueOnce(undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
const cleanupLock = await controller.acquireForCleanup();
|
||||
const disposal = controller.dispose();
|
||||
|
||||
await expect(cleanupLock.release()).rejects.toBe(releaseError);
|
||||
await expect(disposal).resolves.toBeUndefined();
|
||||
expect(release).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("rejects a second cleanup acquisition without replacing ownership", async () => {
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
const cleanupLock = await controller.acquireForCleanup();
|
||||
|
||||
await expect(controller.acquireForCleanup()).rejects.toThrow("attempt cleanup already started");
|
||||
await cleanupLock.release();
|
||||
await expect(controller.dispose()).resolves.toBeUndefined();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("allows cleanup from an async descendant after its write callback settles", async () => {
|
||||
let resumeDescendant!: () => void;
|
||||
const descendantBlocked = new Promise<void>((resolve) => {
|
||||
resumeDescendant = resolve;
|
||||
});
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
let descendant!: Promise<void>;
|
||||
|
||||
await controller.withSessionWriteLock(() => {
|
||||
descendant = (async () => {
|
||||
await descendantBlocked;
|
||||
const cleanupLock = await controller.acquireForCleanup();
|
||||
await cleanupLock.release();
|
||||
})();
|
||||
});
|
||||
resumeDescendant();
|
||||
await descendant;
|
||||
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("lets a started write finish nested writes after disposal begins", async () => {
|
||||
let resumeOuter!: () => void;
|
||||
const outerBlocked = new Promise<void>((resolve) => {
|
||||
resumeOuter = resolve;
|
||||
});
|
||||
const nestedWrite = vi.fn();
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
const outer = controller.withSessionWriteLock(async () => {
|
||||
await outerBlocked;
|
||||
await controller.withSessionWriteLock(nestedWrite);
|
||||
});
|
||||
await Promise.resolve();
|
||||
const disposal = controller.dispose();
|
||||
|
||||
resumeOuter();
|
||||
await Promise.all([outer, disposal]);
|
||||
expect(nestedWrite).toHaveBeenCalledOnce();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps disposal tied to a released prompt until the bounded timeout", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
await controller.releaseForPrompt();
|
||||
let disposed = false;
|
||||
const disposal = controller.dispose().then(() => {
|
||||
disposed = true;
|
||||
});
|
||||
|
||||
await vi.advanceTimersByTimeAsync(4_999);
|
||||
expect(disposed).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await disposal;
|
||||
expect(disposed).toBe(true);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("settles a released prompt when reacquire arrives after disposal", async () => {
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
await controller.releaseForPrompt();
|
||||
const disposal = controller.dispose();
|
||||
|
||||
await controller.reacquireAfterPrompt();
|
||||
await expect(disposal).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps a queued next prompt settlement after the prior reacquire finishes", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let markReloadStarted!: () => void;
|
||||
const reloadStarted = new Promise<void>((resolve) => {
|
||||
markReloadStarted = resolve;
|
||||
});
|
||||
let finishReload!: () => void;
|
||||
const reloadBlocked = new Promise<void>((resolve) => {
|
||||
finishReload = resolve;
|
||||
});
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile: async () => {
|
||||
markReloadStarted();
|
||||
await reloadBlocked;
|
||||
},
|
||||
});
|
||||
|
||||
await controller.releaseForPrompt();
|
||||
const firstReacquire = controller.reacquireAfterPrompt();
|
||||
await reloadStarted;
|
||||
const secondRelease = controller.releaseForPrompt();
|
||||
finishReload();
|
||||
await Promise.all([firstReacquire, secondRelease]);
|
||||
|
||||
let disposed = false;
|
||||
const disposal = controller.dispose().then(() => {
|
||||
disposed = true;
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(4_999);
|
||||
expect(disposed).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await disposal;
|
||||
expect(disposed).toBe(true);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps repeated disposal bounded after abort clears the prompt release", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let markReloadStarted!: () => void;
|
||||
const reloadStarted = new Promise<void>((resolve) => {
|
||||
markReloadStarted = resolve;
|
||||
});
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile: async () => {
|
||||
markReloadStarted();
|
||||
await new Promise<void>(() => {});
|
||||
},
|
||||
});
|
||||
|
||||
await controller.releaseForPrompt();
|
||||
void controller.reacquireAfterPrompt();
|
||||
await reloadStarted;
|
||||
await controller.releaseHeldLockForAbort();
|
||||
const firstDisposal = controller.dispose();
|
||||
const secondDisposal = controller.dispose();
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
|
||||
await expect(Promise.all([firstDisposal, secondDisposal])).resolves.toEqual([
|
||||
undefined,
|
||||
undefined,
|
||||
]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a late SDK prompt handoff after cleanup has started", async () => {
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await controller.acquireForCleanup();
|
||||
await expect(controller.releaseForPrompt()).rejects.toThrow(
|
||||
"attempt cleanup started before prompt submission",
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves the abort reason for a late SDK prompt handoff", async () => {
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
const reason = new Error("cron setup timed out");
|
||||
reason.name = "TimeoutError";
|
||||
|
||||
expect(controller.isPromptSubmissionBlockedError(undefined)).toBe(false);
|
||||
await controller.releaseHeldLockForAbort({ reason });
|
||||
|
||||
expect(controller.isPromptSubmissionBlockedError(reason)).toBe(true);
|
||||
expect(controller.isPromptSubmissionBlockedError(new Error(reason.message))).toBe(false);
|
||||
await expect(controller.releaseForPrompt()).rejects.toBe(reason);
|
||||
});
|
||||
|
||||
it("rejects a late SDK prompt handoff after disposal", async () => {
|
||||
const write = vi.fn();
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await controller.dispose();
|
||||
await controller.dispose();
|
||||
await expect(controller.releaseForPrompt()).rejects.toThrow(
|
||||
"attempt disposed before prompt submission",
|
||||
);
|
||||
await expect(controller.withSessionWriteLock(write)).rejects.toThrow(
|
||||
"attempt disposed before transcript write",
|
||||
);
|
||||
expect(write).not.toHaveBeenCalled();
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("retries direct disposal when the underlying release fails", async () => {
|
||||
const releaseError = new Error("release failed");
|
||||
const release = vi
|
||||
.fn<() => Promise<void>>()
|
||||
.mockRejectedValueOnce(releaseError)
|
||||
.mockResolvedValueOnce(undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
|
||||
await expect(controller.dispose()).rejects.toBe(releaseError);
|
||||
await expect(controller.dispose()).resolves.toBeUndefined();
|
||||
expect(release).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps session ownership until an active write callback settles", async () => {
|
||||
let releaseWrite!: () => void;
|
||||
const writeBlocked = new Promise<void>((resolve) => {
|
||||
releaseWrite = resolve;
|
||||
});
|
||||
const release = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
});
|
||||
const write = controller.withSessionWriteLock(async () => await writeBlocked);
|
||||
await Promise.resolve();
|
||||
let disposeSettled = false;
|
||||
const disposal = controller.dispose().then(() => {
|
||||
disposeSettled = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(disposeSettled).toBe(false);
|
||||
expect(release).not.toHaveBeenCalled();
|
||||
releaseWrite();
|
||||
await Promise.all([write, disposal]);
|
||||
expect(release).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("reloads after a delayed prompt following a non-terminal sessions_yield abort", async () => {
|
||||
const reloadPromptReleasedSessionFile = vi.fn();
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile,
|
||||
});
|
||||
|
||||
await controller.releaseHeldLockForAbort({ terminal: false });
|
||||
await expect(controller.releaseForPrompt()).resolves.toBeUndefined();
|
||||
await expect(controller.reacquireAfterPrompt()).resolves.toBeUndefined();
|
||||
expect(reloadPromptReleasedSessionFile).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("reloads released transcript state during cleanup after a terminal abort", async () => {
|
||||
const reloadPromptReleasedSessionFile = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile,
|
||||
});
|
||||
|
||||
await controller.releaseForPrompt();
|
||||
await controller.releaseHeldLockForAbort();
|
||||
const cleanupLock = await controller.acquireForCleanup();
|
||||
|
||||
expect(reloadPromptReleasedSessionFile).toHaveBeenCalledOnce();
|
||||
await cleanupLock.release();
|
||||
});
|
||||
|
||||
it("rejects cleanup acquisition when disposal wins during prompt reload", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let markReloadStarted!: () => void;
|
||||
const reloadStarted = new Promise<void>((resolve) => {
|
||||
markReloadStarted = resolve;
|
||||
});
|
||||
let finishReload!: () => void;
|
||||
const reloadBlocked = new Promise<void>((resolve) => {
|
||||
finishReload = resolve;
|
||||
});
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile: async () => {
|
||||
markReloadStarted();
|
||||
await reloadBlocked;
|
||||
},
|
||||
});
|
||||
await controller.releaseForPrompt();
|
||||
const cleanupAcquisition = controller.acquireForCleanup();
|
||||
const cleanupOutcome = cleanupAcquisition.then(
|
||||
() => undefined,
|
||||
(error: unknown) => error,
|
||||
);
|
||||
await reloadStarted;
|
||||
|
||||
const disposal = controller.dispose();
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
await expect(disposal).resolves.toBeUndefined();
|
||||
finishReload();
|
||||
|
||||
await expect(cleanupOutcome).resolves.toEqual(
|
||||
expect.objectContaining({ message: "attempt disposed before cleanup" }),
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("lets cleanup settle a completed prompt when the SDK omits reacquire", async () => {
|
||||
const reloadPromptReleasedSessionFile = vi.fn(async () => undefined);
|
||||
const controller = await createEmbeddedAttemptSessionLockController({
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => undefined })),
|
||||
lockOptions: { sessionFile: "agent:main:main" },
|
||||
reloadPromptReleasedSessionFile,
|
||||
});
|
||||
|
||||
await controller.releaseForPrompt();
|
||||
await expect(controller.acquireForCleanup()).resolves.toBeDefined();
|
||||
expect(reloadPromptReleasedSessionFile).toHaveBeenCalledOnce();
|
||||
await expect(controller.reacquireAfterPrompt()).resolves.toBeUndefined();
|
||||
expect(reloadPromptReleasedSessionFile).toHaveBeenCalledOnce();
|
||||
await expect(controller.releaseForPrompt()).rejects.toThrow(
|
||||
"attempt cleanup started before prompt submission",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("installPromptSubmissionLockRelease", () => {
|
||||
it("preserves the provider error when prompt settlement also fails", async () => {
|
||||
const providerError = new Error("provider failed");
|
||||
const settlementError = new Error("reacquire failed");
|
||||
const session = {
|
||||
agent: {
|
||||
streamFn: vi.fn(async () => {
|
||||
throw providerError;
|
||||
}),
|
||||
},
|
||||
};
|
||||
installPromptSubmissionLockRelease({
|
||||
session,
|
||||
releaseForPrompt: vi.fn(async () => undefined),
|
||||
reacquireAfterPrompt: vi.fn(async () => {
|
||||
throw settlementError;
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(session.agent.streamFn()).rejects.toBe(providerError);
|
||||
expect(providerError.cause).toBe(settlementError);
|
||||
});
|
||||
|
||||
it("does not attach a shared provider and settlement error as its own cause", async () => {
|
||||
const sharedError = new Error("shared failure");
|
||||
const session = {
|
||||
agent: {
|
||||
streamFn: vi.fn(async () => {
|
||||
throw sharedError;
|
||||
}),
|
||||
},
|
||||
};
|
||||
installPromptSubmissionLockRelease({
|
||||
session,
|
||||
releaseForPrompt: vi.fn(async () => undefined),
|
||||
reacquireAfterPrompt: vi.fn(async () => {
|
||||
throw sharedError;
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(session.agent.streamFn()).rejects.toBe(sharedError);
|
||||
expect(sharedError.cause).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves an undefined prompt settlement rejection", async () => {
|
||||
const session = { agent: { streamFn: vi.fn(async () => "ok") } };
|
||||
const undefinedRejection = new Promise<void>((_resolve, reject) => {
|
||||
queueMicrotask(() => {
|
||||
Reflect.apply(reject, undefined, [undefined]);
|
||||
});
|
||||
});
|
||||
installPromptSubmissionLockRelease({
|
||||
session,
|
||||
releaseForPrompt: vi.fn(async () => undefined),
|
||||
reacquireAfterPrompt: vi.fn(() => undefinedRejection),
|
||||
});
|
||||
|
||||
await expect(session.agent.streamFn()).rejects.toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves a frozen provider error when prompt settlement also fails", async () => {
|
||||
const providerError = new Error("frozen provider failure");
|
||||
Object.freeze(providerError);
|
||||
const session = {
|
||||
agent: {
|
||||
streamFn: vi.fn(async () => {
|
||||
throw providerError;
|
||||
}),
|
||||
},
|
||||
};
|
||||
installPromptSubmissionLockRelease({
|
||||
session,
|
||||
releaseForPrompt: vi.fn(async () => undefined),
|
||||
reacquireAfterPrompt: vi.fn(async () => {
|
||||
throw new Error("settlement failed");
|
||||
}),
|
||||
});
|
||||
|
||||
await expect(session.agent.streamFn()).rejects.toBe(providerError);
|
||||
});
|
||||
});
|
||||
@@ -1,654 +0,0 @@
|
||||
/** Coordinates embedded-attempt lifecycle around SQLite-owned transcript writes. */
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
import type {
|
||||
OwnedSessionTranscriptCacheSnapshot,
|
||||
OwnedSessionTranscriptWriteOptions,
|
||||
SessionTranscriptWriteLockTarget,
|
||||
} from "../../../config/sessions/transcript-write-context.js";
|
||||
import { withOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js";
|
||||
import { isSessionWriteLockAcquireError } from "../../session-write-lock-error.js";
|
||||
import type { acquireSessionWriteLock } from "../../session-write-lock.js";
|
||||
import { log } from "../logger.js";
|
||||
|
||||
type SessionLock = Awaited<ReturnType<typeof acquireSessionWriteLock>>;
|
||||
type AcquireSessionWriteLock = typeof acquireSessionWriteLock;
|
||||
type SessionKeyLockOptions = Extract<
|
||||
Parameters<AcquireSessionWriteLock>[0],
|
||||
{ targetKind: "session-key" }
|
||||
>;
|
||||
type LockOptions = Omit<SessionKeyLockOptions, "targetKind"> & {
|
||||
targetKind?: "session-key";
|
||||
};
|
||||
const PROMPT_DISPOSE_SETTLE_TIMEOUT_MS = 5_000;
|
||||
// Teardown must release the process-owned lease even when an admitted writer never settles.
|
||||
const SESSION_LOCK_TEARDOWN_BUDGET_MS = 30_000;
|
||||
|
||||
function createPromptSubmissionAbortError(reason: unknown): Error {
|
||||
if (reason instanceof Error) {
|
||||
return reason;
|
||||
}
|
||||
return reason === undefined
|
||||
? new Error("attempt aborted before prompt submission")
|
||||
: new Error("attempt aborted before prompt submission", { cause: reason });
|
||||
}
|
||||
|
||||
export type EmbeddedAttemptSessionFileOwner = {
|
||||
sessionFileKey: string;
|
||||
release(): void;
|
||||
};
|
||||
|
||||
/** Session lanes and SQLite writer queues already serialize this identity. */
|
||||
export async function acquireEmbeddedAttemptSessionFileOwner(params: {
|
||||
sessionFile: string;
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
}): Promise<EmbeddedAttemptSessionFileOwner> {
|
||||
if (params.signal?.aborted) {
|
||||
throw params.signal.reason;
|
||||
}
|
||||
return { sessionFileKey: params.sessionFile, release() {} };
|
||||
}
|
||||
|
||||
export class EmbeddedAttemptSessionTakeoverError extends Error {
|
||||
constructor(sessionKey: string) {
|
||||
super(`session changed while the prompt was running: ${sessionKey}`);
|
||||
this.name = "EmbeddedAttemptSessionTakeoverError";
|
||||
}
|
||||
}
|
||||
|
||||
export type EmbeddedAttemptSessionLockController = {
|
||||
assertOwned(): void;
|
||||
canAdvanceSessionEntryCache(snapshot: OwnedSessionTranscriptCacheSnapshot): boolean;
|
||||
publishOwnedSessionFileSnapshot(snapshot: OwnedSessionTranscriptCacheSnapshot): boolean;
|
||||
publishValidatedSessionFileSnapshot(snapshot: OwnedSessionTranscriptCacheSnapshot): boolean;
|
||||
readTrustedCurrentSessionFileSnapshot(): Promise<undefined>;
|
||||
isPromptSubmissionBlockedError(error: unknown): boolean;
|
||||
releaseForPrompt(): Promise<void>;
|
||||
releaseHeldLockForAbort(options?: { reason?: unknown; terminal?: boolean }): Promise<void>;
|
||||
refreshAfterOwnedSessionWrite(): void;
|
||||
reacquireAfterPrompt(): Promise<void>;
|
||||
withSessionWriteLock<T>(
|
||||
run: () => Promise<T> | T,
|
||||
options?: OwnedSessionTranscriptWriteOptions<T>,
|
||||
): Promise<T>;
|
||||
acquireForCleanup(params?: { session?: unknown }): Promise<SessionLock>;
|
||||
hasSessionTakeover(): boolean;
|
||||
dispose(): Promise<void>;
|
||||
};
|
||||
|
||||
export async function createEmbeddedAttemptSessionLockController(params: {
|
||||
acquireSessionWriteLock: AcquireSessionWriteLock;
|
||||
initialAcquireSignal?: AbortSignal;
|
||||
lockOptions: LockOptions;
|
||||
runId?: string;
|
||||
sessionId?: string;
|
||||
reloadPromptReleasedSessionFile?: () => Promise<void> | void;
|
||||
}): Promise<EmbeddedAttemptSessionLockController> {
|
||||
// The runtime caller supplies resolveSessionWriteLockTargetKey(sessionTarget),
|
||||
// so aliases for one session incarnation converge before lease acquisition.
|
||||
const noOpLock = await params.acquireSessionWriteLock({
|
||||
...params.lockOptions,
|
||||
targetKind: "session-key",
|
||||
...(params.initialAcquireSignal ? { signal: params.initialAcquireSignal } : {}),
|
||||
});
|
||||
let initialLockReleasePromise: Promise<void> | undefined;
|
||||
let initialLockReleased = false;
|
||||
const releaseInitialLock = (): Promise<void> => {
|
||||
if (initialLockReleased) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
initialLockReleasePromise ??= Promise.resolve(noOpLock.release()).then(
|
||||
() => {
|
||||
initialLockReleased = true;
|
||||
},
|
||||
(error: unknown) => {
|
||||
initialLockReleasePromise = undefined;
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
return initialLockReleasePromise;
|
||||
};
|
||||
let disposed = false;
|
||||
let promptAborted = false;
|
||||
let promptSubmissionBlockedError: Error | undefined;
|
||||
let takeoverDetected = false;
|
||||
const assertInitialLockOwned = (): void => {
|
||||
try {
|
||||
noOpLock.assertOwned?.();
|
||||
} catch (error) {
|
||||
if (isSessionWriteLockAcquireError(error)) {
|
||||
takeoverDetected = true;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
let promptReleaseNeedsReload = false;
|
||||
let cleanupRequested = false;
|
||||
let promptSettled = Promise.resolve();
|
||||
let settlePrompt: (() => void) | undefined;
|
||||
let lifecycle = Promise.resolve();
|
||||
let reloadFailed = false;
|
||||
let reloadFailure: unknown;
|
||||
let disposePromise: Promise<void> | undefined;
|
||||
let cleanupReleasePromise: Promise<void> | undefined;
|
||||
let cleanupLockGranted = false;
|
||||
let cleanupOwnershipReleased: Promise<void> | undefined;
|
||||
let resolveCleanupOwnershipReleased: (() => void) | undefined;
|
||||
let rejectCleanupOwnershipReleased: ((error: unknown) => void) | undefined;
|
||||
type ActiveWriteOperation = {
|
||||
active: boolean;
|
||||
settlement?: Promise<void>;
|
||||
started: boolean;
|
||||
};
|
||||
const activeWriteOperations = new Set<ActiveWriteOperation>();
|
||||
const activeWriteOperation = new AsyncLocalStorage<ActiveWriteOperation>();
|
||||
type LifecycleOwner = {
|
||||
active: boolean;
|
||||
nestedPending: number;
|
||||
nestedTail: Promise<void>;
|
||||
pendingOperations: Set<Promise<void>>;
|
||||
};
|
||||
const createLifecycleOwner = (): LifecycleOwner => ({
|
||||
active: true,
|
||||
nestedPending: 0,
|
||||
nestedTail: Promise.resolve(),
|
||||
pendingOperations: new Set(),
|
||||
});
|
||||
const lifecycleOwner = new AsyncLocalStorage<LifecycleOwner>();
|
||||
let teardownBudgetLogged = false;
|
||||
const createTeardownDeadline = (budgetMs = SESSION_LOCK_TEARDOWN_BUDGET_MS): number =>
|
||||
performance.now() + budgetMs;
|
||||
const pendingWriteCount = (): number => activeWriteOperations.size;
|
||||
const logTeardownBudgetExpiry = (phase: string): void => {
|
||||
if (teardownBudgetLogged) {
|
||||
return;
|
||||
}
|
||||
teardownBudgetLogged = true;
|
||||
log.error(
|
||||
`session lock teardown budget expired; releasing ownership: ` +
|
||||
`phase=${phase} runId=${params.runId ?? "unknown"} ` +
|
||||
`sessionId=${params.sessionId ?? "unknown"} ` +
|
||||
`pendingWrites=${pendingWriteCount()} timeoutMs=${SESSION_LOCK_TEARDOWN_BUDGET_MS}`,
|
||||
);
|
||||
};
|
||||
const settleWithinTeardownDeadline = async (
|
||||
operation: Promise<void>,
|
||||
deadline: number,
|
||||
): Promise<boolean> => {
|
||||
const waitMs = Math.max(0, deadline - performance.now());
|
||||
if (waitMs <= 0) {
|
||||
void operation.catch(() => {});
|
||||
return false;
|
||||
}
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
operation.then(() => true),
|
||||
new Promise<false>((resolve) => {
|
||||
timeout = setTimeout(() => resolve(false), waitMs);
|
||||
timeout.unref?.();
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
};
|
||||
const drainActiveWrites = async (startedOnly: boolean): Promise<void> => {
|
||||
while (true) {
|
||||
const pendingWrites = [...activeWriteOperations]
|
||||
.filter((operation) => !startedOnly || operation.started)
|
||||
.flatMap((operation) => (operation.settlement ? [operation.settlement] : []));
|
||||
if (pendingWrites.length === 0) {
|
||||
return;
|
||||
}
|
||||
await Promise.all(pendingWrites);
|
||||
}
|
||||
};
|
||||
const drainLifecycleOwner = async (owner: LifecycleOwner): Promise<void> => {
|
||||
let failed = false;
|
||||
let firstError: unknown;
|
||||
while (owner.pendingOperations.size > 0) {
|
||||
const pending = [...owner.pendingOperations];
|
||||
owner.pendingOperations.clear();
|
||||
const settled = await Promise.allSettled(pending);
|
||||
const rejection = settled.find((result) => result.status === "rejected");
|
||||
if (!failed && rejection?.status === "rejected") {
|
||||
failed = true;
|
||||
firstError = rejection.reason;
|
||||
}
|
||||
}
|
||||
if (failed) {
|
||||
throw firstError;
|
||||
}
|
||||
};
|
||||
const runLifecycleOwner = async <T>(
|
||||
owner: LifecycleOwner,
|
||||
run: () => Promise<T> | T,
|
||||
): Promise<T> => {
|
||||
let value: T | undefined;
|
||||
let primaryError: unknown;
|
||||
let primaryFailed = false;
|
||||
try {
|
||||
value = await lifecycleOwner.run(owner, async () => await run());
|
||||
} catch (error) {
|
||||
primaryFailed = true;
|
||||
primaryError = error;
|
||||
}
|
||||
let drainError: unknown;
|
||||
let drainFailed = false;
|
||||
try {
|
||||
await drainLifecycleOwner(owner);
|
||||
} catch (error) {
|
||||
drainFailed = true;
|
||||
drainError = error;
|
||||
} finally {
|
||||
owner.active = false;
|
||||
}
|
||||
if (primaryFailed) {
|
||||
if (
|
||||
drainFailed &&
|
||||
drainError !== primaryError &&
|
||||
primaryError instanceof Error &&
|
||||
primaryError.cause === undefined
|
||||
) {
|
||||
try {
|
||||
primaryError.cause = drainError;
|
||||
} catch {
|
||||
// Frozen callback errors remain primary; drain failure is secondary.
|
||||
}
|
||||
}
|
||||
throw primaryError;
|
||||
}
|
||||
if (drainFailed) {
|
||||
throw drainError;
|
||||
}
|
||||
return value as T;
|
||||
};
|
||||
const serializeLifecycle = async <T>(run: () => Promise<T> | T): Promise<T> => {
|
||||
const inheritedOwner = lifecycleOwner.getStore();
|
||||
if (inheritedOwner?.active) {
|
||||
const previousNested = inheritedOwner.nestedTail;
|
||||
const waitForPrevious = inheritedOwner.nestedPending > 0 ? previousNested : Promise.resolve();
|
||||
inheritedOwner.nestedPending += 1;
|
||||
const operation = waitForPrevious.then(async () => {
|
||||
const childOwner = createLifecycleOwner();
|
||||
return await runLifecycleOwner(childOwner, run);
|
||||
});
|
||||
void operation.catch(() => {});
|
||||
const queueTail = operation.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
const propagated = operation.then(() => undefined);
|
||||
void propagated.catch(() => {});
|
||||
inheritedOwner.nestedTail = queueTail;
|
||||
inheritedOwner.pendingOperations.add(propagated);
|
||||
void queueTail.finally(() => {
|
||||
inheritedOwner.nestedPending -= 1;
|
||||
});
|
||||
return await operation;
|
||||
}
|
||||
const previous = lifecycle;
|
||||
let release!: () => void;
|
||||
lifecycle = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
await previous;
|
||||
const owner = createLifecycleOwner();
|
||||
try {
|
||||
return await runLifecycleOwner(owner, run);
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
};
|
||||
const reloadPromptReleasedState = async (): Promise<void> => {
|
||||
if (reloadFailed) {
|
||||
throw reloadFailure;
|
||||
}
|
||||
try {
|
||||
assertInitialLockOwned();
|
||||
await params.reloadPromptReleasedSessionFile?.();
|
||||
} catch (error) {
|
||||
reloadFailed = true;
|
||||
reloadFailure = error;
|
||||
if (error instanceof EmbeddedAttemptSessionTakeoverError) {
|
||||
takeoverDetected = true;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const settlePromptRelease = (): void => {
|
||||
promptReleaseNeedsReload = false;
|
||||
settlePrompt?.();
|
||||
settlePrompt = undefined;
|
||||
};
|
||||
return {
|
||||
assertOwned: assertInitialLockOwned,
|
||||
canAdvanceSessionEntryCache: () => false,
|
||||
publishOwnedSessionFileSnapshot: () => false,
|
||||
publishValidatedSessionFileSnapshot: () => false,
|
||||
readTrustedCurrentSessionFileSnapshot: async () => undefined,
|
||||
releaseForPrompt: async () => {
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before prompt submission");
|
||||
}
|
||||
await serializeLifecycle(() => {
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before prompt submission");
|
||||
}
|
||||
if (promptSubmissionBlockedError) {
|
||||
throw promptSubmissionBlockedError;
|
||||
}
|
||||
if (cleanupRequested) {
|
||||
throw new Error("attempt cleanup started before prompt submission");
|
||||
}
|
||||
assertInitialLockOwned();
|
||||
// The SQLite lease spans the provider prompt. Its stale deadline never
|
||||
// permits takeover from a live process owner.
|
||||
promptAborted = false;
|
||||
promptReleaseNeedsReload = true;
|
||||
promptSettled = new Promise<void>((resolve) => {
|
||||
settlePrompt = resolve;
|
||||
});
|
||||
});
|
||||
},
|
||||
isPromptSubmissionBlockedError: (error) =>
|
||||
promptSubmissionBlockedError !== undefined && error === promptSubmissionBlockedError,
|
||||
releaseHeldLockForAbort: async (options) => {
|
||||
promptAborted = true;
|
||||
if (options?.terminal !== false && !promptSubmissionBlockedError) {
|
||||
promptSubmissionBlockedError = createPromptSubmissionAbortError(options?.reason);
|
||||
}
|
||||
},
|
||||
refreshAfterOwnedSessionWrite: () => {},
|
||||
reacquireAfterPrompt: async () => {
|
||||
if (disposed) {
|
||||
settlePromptRelease();
|
||||
return;
|
||||
}
|
||||
await serializeLifecycle(async () => {
|
||||
try {
|
||||
if (disposed || cleanupRequested || (promptAborted && !promptReleaseNeedsReload)) {
|
||||
return;
|
||||
}
|
||||
await reloadPromptReleasedState();
|
||||
} finally {
|
||||
settlePromptRelease();
|
||||
}
|
||||
});
|
||||
},
|
||||
withSessionWriteLock: (run) => {
|
||||
const rejectWrite = (error: Error): Promise<never> => {
|
||||
const rejected = Promise.reject(error);
|
||||
void rejected.catch(() => {});
|
||||
return rejected;
|
||||
};
|
||||
const parentWriteOperation = activeWriteOperation.getStore();
|
||||
const activeDescendant = Boolean(
|
||||
parentWriteOperation?.active && activeWriteOperations.has(parentWriteOperation),
|
||||
);
|
||||
if (disposed && !activeDescendant) {
|
||||
return rejectWrite(new Error("attempt disposed before transcript write"));
|
||||
}
|
||||
if (cleanupRequested && !activeDescendant) {
|
||||
return rejectWrite(new Error("attempt cleanup started before transcript write"));
|
||||
}
|
||||
const writeOperation: ActiveWriteOperation = { active: true, started: false };
|
||||
const operation = serializeLifecycle(async () => {
|
||||
if (disposed && !activeDescendant) {
|
||||
throw new Error("attempt disposed before transcript write");
|
||||
}
|
||||
if (cleanupRequested && !activeDescendant) {
|
||||
throw new Error("attempt cleanup started before transcript write");
|
||||
}
|
||||
if (reloadFailed) {
|
||||
throw reloadFailure;
|
||||
}
|
||||
assertInitialLockOwned();
|
||||
writeOperation.started = true;
|
||||
return await activeWriteOperation.run(writeOperation, async () => await run());
|
||||
});
|
||||
const settlement = operation.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
void operation.catch(() => {});
|
||||
writeOperation.settlement = settlement;
|
||||
activeWriteOperations.add(writeOperation);
|
||||
void settlement.then(() => {
|
||||
writeOperation.active = false;
|
||||
activeWriteOperations.delete(writeOperation);
|
||||
});
|
||||
return operation;
|
||||
},
|
||||
acquireForCleanup: async () => {
|
||||
const currentWriteOperation = activeWriteOperation.getStore();
|
||||
if (currentWriteOperation?.active && activeWriteOperations.has(currentWriteOperation)) {
|
||||
throw new Error("cannot start attempt cleanup inside a transcript write callback");
|
||||
}
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before cleanup");
|
||||
}
|
||||
if (cleanupRequested) {
|
||||
throw new Error("attempt cleanup already started");
|
||||
}
|
||||
cleanupRequested = true;
|
||||
const deadline = createTeardownDeadline();
|
||||
const cleanupAdmission = (async () => {
|
||||
await serializeLifecycle(async () => {
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before cleanup");
|
||||
}
|
||||
if (promptReleaseNeedsReload) {
|
||||
try {
|
||||
if (!disposed) {
|
||||
await reloadPromptReleasedState();
|
||||
}
|
||||
} finally {
|
||||
settlePromptRelease();
|
||||
}
|
||||
}
|
||||
});
|
||||
await serializeLifecycle(() => {});
|
||||
})();
|
||||
if (!(await settleWithinTeardownDeadline(cleanupAdmission, deadline))) {
|
||||
disposed = true;
|
||||
promptAborted = true;
|
||||
takeoverDetected = true;
|
||||
settlePromptRelease();
|
||||
logTeardownBudgetExpiry("cleanup-acquire");
|
||||
await releaseInitialLock();
|
||||
cleanupLockGranted = true;
|
||||
cleanupOwnershipReleased = new Promise<void>((resolve) => {
|
||||
resolveCleanupOwnershipReleased = resolve;
|
||||
});
|
||||
return {
|
||||
release: () => {
|
||||
resolveCleanupOwnershipReleased?.();
|
||||
return Promise.resolve();
|
||||
},
|
||||
} as SessionLock;
|
||||
}
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before cleanup");
|
||||
}
|
||||
assertInitialLockOwned();
|
||||
cleanupLockGranted = true;
|
||||
cleanupOwnershipReleased = new Promise<void>((resolve, reject) => {
|
||||
resolveCleanupOwnershipReleased = resolve;
|
||||
rejectCleanupOwnershipReleased = reject;
|
||||
});
|
||||
void cleanupOwnershipReleased.catch(() => {});
|
||||
return {
|
||||
release: () => {
|
||||
cleanupReleasePromise ??= (async () => {
|
||||
try {
|
||||
const writesSettled = await settleWithinTeardownDeadline(
|
||||
drainActiveWrites(false),
|
||||
createTeardownDeadline(),
|
||||
);
|
||||
if (!writesSettled) {
|
||||
logTeardownBudgetExpiry("cleanup-release");
|
||||
}
|
||||
await releaseInitialLock();
|
||||
resolveCleanupOwnershipReleased?.();
|
||||
} catch (error) {
|
||||
rejectCleanupOwnershipReleased?.(error);
|
||||
throw error;
|
||||
}
|
||||
})();
|
||||
return cleanupReleasePromise;
|
||||
},
|
||||
} as SessionLock;
|
||||
},
|
||||
hasSessionTakeover: () => takeoverDetected,
|
||||
dispose: async () => {
|
||||
const currentWriteOperation = activeWriteOperation.getStore();
|
||||
if (currentWriteOperation?.active && activeWriteOperations.has(currentWriteOperation)) {
|
||||
throw new Error("cannot dispose an attempt from inside a transcript write callback");
|
||||
}
|
||||
disposePromise ??= (async () => {
|
||||
disposed = true;
|
||||
promptAborted = true;
|
||||
if (cleanupLockGranted) {
|
||||
// Cleanup acquisition already drained the lifecycle queue. Its lock
|
||||
// release resolves this handoff after any tracked write settlements.
|
||||
try {
|
||||
await cleanupOwnershipReleased;
|
||||
} catch {
|
||||
await releaseInitialLock();
|
||||
}
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await settleWithinTeardownDeadline(
|
||||
(async () => {
|
||||
await Promise.all([promptSettled, serializeLifecycle(() => {})]);
|
||||
await drainActiveWrites(false);
|
||||
})(),
|
||||
createTeardownDeadline(PROMPT_DISPOSE_SETTLE_TIMEOUT_MS),
|
||||
);
|
||||
if (
|
||||
!(await settleWithinTeardownDeadline(drainActiveWrites(true), createTeardownDeadline()))
|
||||
) {
|
||||
logTeardownBudgetExpiry("dispose");
|
||||
}
|
||||
} finally {
|
||||
await releaseInitialLock();
|
||||
}
|
||||
})().catch((error: unknown) => {
|
||||
disposePromise = undefined;
|
||||
throw error;
|
||||
});
|
||||
await disposePromise;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type PromptReleaseStreamFn = ((...args: unknown[]) => Promise<unknown>) & {
|
||||
openclawSessionLockPromptReleaseInstalled?: true;
|
||||
};
|
||||
|
||||
type SessionWithAgentPrompt = {
|
||||
agent?: { streamFn?: PromptReleaseStreamFn };
|
||||
};
|
||||
|
||||
async function settlePromptSubmission(params: {
|
||||
reacquireAfterPrompt: () => Promise<void>;
|
||||
}): Promise<void> {
|
||||
await params.reacquireAfterPrompt();
|
||||
}
|
||||
|
||||
function attachPromptSettlementError(promptError: unknown, settlementError: unknown): void {
|
||||
if (
|
||||
promptError instanceof Error &&
|
||||
promptError.cause === undefined &&
|
||||
settlementError !== promptError
|
||||
) {
|
||||
try {
|
||||
promptError.cause = settlementError;
|
||||
} catch {
|
||||
// A frozen provider error remains the primary failure; settlement diagnostics are secondary.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function installPromptSubmissionLockRelease(params: {
|
||||
session: unknown;
|
||||
releaseForPrompt: () => Promise<void>;
|
||||
reacquireAfterPrompt: () => Promise<void>;
|
||||
sessionFile?: string;
|
||||
sessionKey?: string;
|
||||
sessionTarget?: SessionTranscriptWriteLockTarget;
|
||||
withSessionWriteLock?: <T>(
|
||||
run: () => Promise<T> | T,
|
||||
options?: OwnedSessionTranscriptWriteOptions<T>,
|
||||
) => Promise<T>;
|
||||
assertSessionWriteLockOwned?: () => void;
|
||||
canAdvanceSessionEntryCache?: (snapshot: OwnedSessionTranscriptCacheSnapshot) => boolean;
|
||||
publishSessionFileSnapshot?: (snapshot: OwnedSessionTranscriptCacheSnapshot) => boolean;
|
||||
}): void {
|
||||
const agent = (params.session as SessionWithAgentPrompt).agent;
|
||||
if (typeof agent?.streamFn !== "function") {
|
||||
return;
|
||||
}
|
||||
const currentStreamFn = agent.streamFn;
|
||||
if (currentStreamFn.openclawSessionLockPromptReleaseInstalled === true) {
|
||||
return;
|
||||
}
|
||||
const originalStreamFn = currentStreamFn.bind(agent);
|
||||
const wrappedStreamFn: PromptReleaseStreamFn = async (...args: unknown[]) => {
|
||||
// The internal agent runtime routes transcript mutations through the
|
||||
// lifecycle lock; it has no separate SDK event queue to drain.
|
||||
await params.releaseForPrompt();
|
||||
let promptFailed = false;
|
||||
let promptError: unknown;
|
||||
let promptResult: unknown;
|
||||
try {
|
||||
if (params.sessionFile && params.withSessionWriteLock) {
|
||||
if (!params.assertSessionWriteLockOwned) {
|
||||
throw new Error("owned transcript write context requires a session lock ownership fence");
|
||||
}
|
||||
promptResult = await withOwnedSessionTranscriptWrites(
|
||||
{
|
||||
sessionFile: params.sessionFile,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionTarget: params.sessionTarget,
|
||||
assertOwned: params.assertSessionWriteLockOwned,
|
||||
withSessionWriteLock: params.withSessionWriteLock,
|
||||
canAdvanceSessionEntryCache: params.canAdvanceSessionEntryCache,
|
||||
publishSessionFileSnapshot: params.publishSessionFileSnapshot,
|
||||
},
|
||||
async () => await originalStreamFn(...args),
|
||||
);
|
||||
} else {
|
||||
promptResult = await originalStreamFn(...args);
|
||||
}
|
||||
} catch (error) {
|
||||
promptFailed = true;
|
||||
promptError = error;
|
||||
}
|
||||
let settlementFailed = false;
|
||||
let settlementError: unknown;
|
||||
try {
|
||||
await settlePromptSubmission(params);
|
||||
} catch (error) {
|
||||
settlementFailed = true;
|
||||
settlementError = error;
|
||||
}
|
||||
if (promptFailed) {
|
||||
if (settlementFailed) {
|
||||
attachPromptSettlementError(promptError, settlementError);
|
||||
}
|
||||
throw promptError;
|
||||
}
|
||||
if (settlementFailed) {
|
||||
throw settlementError;
|
||||
}
|
||||
return promptResult;
|
||||
};
|
||||
wrappedStreamFn.openclawSessionLockPromptReleaseInstalled = true;
|
||||
agent.streamFn = wrappedStreamFn;
|
||||
}
|
||||
@@ -3233,8 +3233,7 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("releases the session lock even when teardown cleanup throws", async () => {
|
||||
const releaseMock = vi.fn(async () => {});
|
||||
it("disposes the session even when teardown cleanup throws", async () => {
|
||||
const disposeMock = vi.fn();
|
||||
const flushMock = vi.fn(async () => {
|
||||
throw new Error("flush failed");
|
||||
@@ -3246,12 +3245,10 @@ describe("runEmbeddedAttempt context engine sessionKey forwarding", () => {
|
||||
session: { agent: {}, dispose: disposeMock },
|
||||
sessionManager: hoisted.sessionManager,
|
||||
bundleLspRuntime: undefined,
|
||||
sessionLock: { release: releaseMock },
|
||||
});
|
||||
|
||||
expect(flushMock).toHaveBeenCalledTimes(1);
|
||||
expect(disposeMock).toHaveBeenCalledTimes(1);
|
||||
expect(releaseMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -36,8 +36,6 @@ import type { WorkspaceBootstrapFile } from "../../workspace.js";
|
||||
|
||||
type SubscribeEmbeddedAgentSessionFn =
|
||||
typeof import("../../embedded-agent-subscribe.js").subscribeEmbeddedAgentSession;
|
||||
type AcquireSessionWriteLockFn =
|
||||
typeof import("../../session-write-lock.js").acquireSessionWriteLock;
|
||||
type ShouldPreemptivelyCompactBeforePromptFn =
|
||||
typeof import("./preemptive-compaction.js").shouldPreemptivelyCompactBeforePrompt;
|
||||
|
||||
@@ -95,7 +93,6 @@ type AttemptSpawnWorkspaceHoisted = {
|
||||
buildEmbeddedMessageActionDiscoveryInputMock: UnknownMock;
|
||||
createOpenClawCodingToolsMock: UnknownMock;
|
||||
subscribeEmbeddedAgentSessionMock: Mock<SubscribeEmbeddedAgentSessionFn>;
|
||||
acquireSessionWriteLockMock: Mock<AcquireSessionWriteLockFn>;
|
||||
installToolResultContextGuardMock: UnknownMock;
|
||||
installContextEngineLoopHookMock: UnknownMock;
|
||||
flushPendingToolResultsAfterIdleMock: AsyncUnknownMock;
|
||||
@@ -186,9 +183,6 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => {
|
||||
const subscribeEmbeddedAgentSessionMock = vi.fn<SubscribeEmbeddedAgentSessionFn>(() =>
|
||||
createSubscriptionMock(),
|
||||
);
|
||||
const acquireSessionWriteLockMock = vi.fn<AcquireSessionWriteLockFn>(async (_params) => ({
|
||||
release: async () => {},
|
||||
}));
|
||||
const resolveBootstrapContextForRunMock = vi.fn<() => Promise<BootstrapContext>>(async () => ({
|
||||
bootstrapFiles: [],
|
||||
contextFiles: [],
|
||||
@@ -266,7 +260,6 @@ const hoisted = vi.hoisted((): AttemptSpawnWorkspaceHoisted => {
|
||||
buildEmbeddedMessageActionDiscoveryInputMock,
|
||||
createOpenClawCodingToolsMock,
|
||||
subscribeEmbeddedAgentSessionMock,
|
||||
acquireSessionWriteLockMock,
|
||||
installToolResultContextGuardMock,
|
||||
installContextEngineLoopHookMock,
|
||||
flushPendingToolResultsAfterIdleMock,
|
||||
@@ -565,20 +558,6 @@ vi.mock("../tool-schema-runtime.js", () => ({
|
||||
normalizeProviderToolSchemas: ({ tools }: { tools: unknown[] }) => tools,
|
||||
}));
|
||||
|
||||
vi.mock("../../session-write-lock.js", async () => {
|
||||
const { resolveSessionWriteLockTargetKey } = await vi.importActual<
|
||||
typeof import("../../session-write-lock.js")
|
||||
>("../../session-write-lock.js");
|
||||
return {
|
||||
acquireSessionWriteLock: (params: Parameters<AcquireSessionWriteLockFn>[0]) =>
|
||||
hoisted.acquireSessionWriteLockMock(params),
|
||||
resolveSessionWriteLockAcquireTimeoutMs: () => 60000,
|
||||
resolveSessionWriteLockOptions: () => ({ timeoutMs: 60000, staleMs: 1_800_000, maxHoldMs: 1 }),
|
||||
resolveSessionLockMaxHoldFromTimeout: () => 1,
|
||||
resolveSessionWriteLockTargetKey,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../tool-result-context-guard.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../tool-result-context-guard.js")>(
|
||||
"../tool-result-context-guard.js",
|
||||
@@ -1090,9 +1069,6 @@ export function resetEmbeddedAttemptHarness(
|
||||
hoisted.subscribeEmbeddedAgentSessionMock
|
||||
.mockReset()
|
||||
.mockImplementation(() => createSubscriptionMock());
|
||||
hoisted.acquireSessionWriteLockMock.mockReset().mockResolvedValue({
|
||||
release: async () => {},
|
||||
});
|
||||
hoisted.installToolResultContextGuardMock.mockReset().mockReturnValue(() => {});
|
||||
hoisted.installContextEngineLoopHookMock.mockReset().mockReturnValue(() => {});
|
||||
hoisted.flushPendingToolResultsAfterIdleMock.mockReset().mockResolvedValue(undefined);
|
||||
|
||||
@@ -11,7 +11,7 @@ describe("cleanupEmbeddedAttemptResources", () => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("waits for aborted prompt settlement before flushing and releasing the lock", async () => {
|
||||
it("waits for aborted prompt settlement before flushing and disposing", async () => {
|
||||
// After an abort, pending prompt work gets a short chance to settle before
|
||||
// session flush/release/dispose run.
|
||||
const order: string[] = [];
|
||||
@@ -31,11 +31,6 @@ describe("cleanupEmbeddedAttemptResources", () => {
|
||||
},
|
||||
},
|
||||
sessionManager: {},
|
||||
sessionLock: {
|
||||
release: async () => {
|
||||
order.push("release");
|
||||
},
|
||||
},
|
||||
aborted: true,
|
||||
abortSettlePromise: settle.promise,
|
||||
runId: "run-1",
|
||||
@@ -49,10 +44,10 @@ describe("cleanupEmbeddedAttemptResources", () => {
|
||||
settle.resolve();
|
||||
await cleanupPromise;
|
||||
|
||||
expect(order).toEqual(["guard", "flush", "release", "dispose"]);
|
||||
expect(order).toEqual(["guard", "flush", "dispose"]);
|
||||
});
|
||||
|
||||
it("releases the lock after the aborted settle timeout", async () => {
|
||||
it("continues cleanup after the aborted settle timeout", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.spyOn(log, "warn").mockImplementation(() => {});
|
||||
const order: string[] = [];
|
||||
@@ -68,11 +63,6 @@ describe("cleanupEmbeddedAttemptResources", () => {
|
||||
},
|
||||
},
|
||||
sessionManager: {},
|
||||
sessionLock: {
|
||||
release: async () => {
|
||||
order.push("release");
|
||||
},
|
||||
},
|
||||
aborted: true,
|
||||
abortSettlePromise: new Promise(() => {}),
|
||||
runId: "run-1",
|
||||
@@ -86,12 +76,10 @@ describe("cleanupEmbeddedAttemptResources", () => {
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await cleanupPromise;
|
||||
|
||||
expect(order).toEqual(["flush", "release", "dispose"]);
|
||||
expect(order).toEqual(["flush", "dispose"]);
|
||||
});
|
||||
|
||||
it("releases the lock before runtime teardown can hang", async () => {
|
||||
// Bundle runtime disposal can hang; release transcript locks first so other
|
||||
// turns are not blocked by diagnostic cleanup.
|
||||
it("disposes the session before runtime teardown can hang", async () => {
|
||||
const order: string[] = [];
|
||||
let markRuntimeDisposeStarted!: () => void;
|
||||
const runtimeDisposeStarted = new Promise<void>((resolve) => {
|
||||
@@ -109,11 +97,6 @@ describe("cleanupEmbeddedAttemptResources", () => {
|
||||
},
|
||||
},
|
||||
sessionManager: {},
|
||||
sessionLock: {
|
||||
release: async () => {
|
||||
order.push("release");
|
||||
},
|
||||
},
|
||||
bundleMcpRuntime: {
|
||||
dispose: async () => {
|
||||
order.push("runtime-dispose-start");
|
||||
@@ -125,81 +108,25 @@ describe("cleanupEmbeddedAttemptResources", () => {
|
||||
|
||||
await runtimeDisposeStarted;
|
||||
|
||||
expect(order).toEqual(["flush", "release", "dispose", "runtime-dispose-start"]);
|
||||
expect(order).toEqual(["flush", "dispose", "runtime-dispose-start"]);
|
||||
});
|
||||
|
||||
it("does not wait for the settle promise on non-aborted cleanup", async () => {
|
||||
const release = vi.fn(async () => {});
|
||||
const dispose = vi.fn();
|
||||
|
||||
await cleanupEmbeddedAttemptResources({
|
||||
flushPendingToolResultsAfterIdle: vi.fn(async () => {}),
|
||||
session: {
|
||||
agent: {},
|
||||
dispose: vi.fn(),
|
||||
dispose,
|
||||
},
|
||||
sessionManager: {},
|
||||
sessionLock: { release },
|
||||
aborted: false,
|
||||
abortSettlePromise: new Promise(() => {}),
|
||||
runId: "run-1",
|
||||
sessionId: "session-1",
|
||||
});
|
||||
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("still disposes resources when lock release fails", async () => {
|
||||
const releaseError = new Error("release failed");
|
||||
const dispose = vi.fn();
|
||||
const runtimeDispose = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
cleanupEmbeddedAttemptResources({
|
||||
flushPendingToolResultsAfterIdle: vi.fn(async () => {}),
|
||||
session: {
|
||||
agent: {},
|
||||
dispose,
|
||||
},
|
||||
sessionManager: {},
|
||||
sessionLock: {
|
||||
release: async () => {
|
||||
throw releaseError;
|
||||
},
|
||||
},
|
||||
bundleMcpRuntime: {
|
||||
dispose: runtimeDispose,
|
||||
},
|
||||
}),
|
||||
).rejects.toBe(releaseError);
|
||||
|
||||
expect(dispose).toHaveBeenCalledTimes(1);
|
||||
expect(runtimeDispose).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("can skip stale session-manager flushing after session takeover", async () => {
|
||||
const flushPendingToolResultsAfterIdle = vi.fn(async () => {});
|
||||
const order: string[] = [];
|
||||
const dispose = vi.fn(() => {
|
||||
order.push("dispose");
|
||||
});
|
||||
const release = vi.fn(async () => {
|
||||
order.push("release");
|
||||
});
|
||||
|
||||
await cleanupEmbeddedAttemptResources({
|
||||
flushPendingToolResultsAfterIdle,
|
||||
session: {
|
||||
agent: {},
|
||||
dispose,
|
||||
},
|
||||
sessionManager: {},
|
||||
sessionLock: { release },
|
||||
skipSessionFlush: true,
|
||||
});
|
||||
|
||||
expect(flushPendingToolResultsAfterIdle).not.toHaveBeenCalled();
|
||||
expect(dispose).toHaveBeenCalledTimes(1);
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
expect(order).toEqual(["release", "dispose"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* Builds subscription params and cleans up embedded attempt resources.
|
||||
*/
|
||||
import { toErrorObject } from "../../../infra/errors.js";
|
||||
import type { SubscribeEmbeddedAgentSessionParams } from "../../embedded-agent-subscribe.types.js";
|
||||
import { log } from "../logger.js";
|
||||
import { resolveEmbeddedAbortSettleTimeoutMs } from "./attempt.abort-settle-timeout.js";
|
||||
@@ -29,7 +28,7 @@ async function waitForEmbeddedAbortSettle(params: {
|
||||
|
||||
let timeout: NodeJS.Timeout | undefined;
|
||||
// Abort settlement is advisory cleanup; timeout or errors are logged but do
|
||||
// not block releasing the session write-lock.
|
||||
// not block disposing attempt-owned resources.
|
||||
const outcome = await Promise.race([
|
||||
params.promise
|
||||
.then(() => "settled" as const)
|
||||
@@ -65,10 +64,8 @@ export function buildEmbeddedSubscriptionParams(
|
||||
}
|
||||
|
||||
/**
|
||||
* Tears down per-attempt resources in lock-safe order: remove guards, settle
|
||||
* aborted prompts, flush tool results, release the session lock, then dispose
|
||||
* runtimes. Lock release errors are reported after best-effort disposal so a
|
||||
* failed lock does not leak spawned runtimes.
|
||||
* Tears down per-attempt resources after the transcript lifecycle has drained:
|
||||
* remove guards, settle aborted prompts, flush tool results, then dispose runtimes.
|
||||
*/
|
||||
export async function cleanupEmbeddedAttemptResources(params: {
|
||||
removeToolResultContextGuard?: () => void;
|
||||
@@ -81,49 +78,31 @@ export async function cleanupEmbeddedAttemptResources(params: {
|
||||
sessionManager: unknown;
|
||||
bundleMcpRuntime?: { dispose(): Promise<void> | void };
|
||||
bundleLspRuntime?: { dispose(): Promise<void> | void };
|
||||
sessionLock: { release(): Promise<void> | void };
|
||||
aborted?: boolean;
|
||||
abortSettlePromise?: Promise<unknown> | null;
|
||||
skipSessionFlush?: boolean;
|
||||
runId?: string;
|
||||
sessionId?: string;
|
||||
}): Promise<void> {
|
||||
let sessionLockReleaseError: unknown;
|
||||
try {
|
||||
try {
|
||||
params.removeToolResultContextGuard?.();
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
if (params.aborted && params.abortSettlePromise) {
|
||||
await waitForEmbeddedAbortSettle({
|
||||
promise: params.abortSettlePromise,
|
||||
runId: params.runId ?? "unknown",
|
||||
sessionId: params.sessionId ?? "unknown",
|
||||
});
|
||||
}
|
||||
// PERF: When the run was aborted (user stop / timeout), skip the expensive
|
||||
// waitForIdle (up to 30 s) and flush pending tool results synchronously so
|
||||
// the session write-lock is released without leaving orphaned tool calls.
|
||||
if (!params.skipSessionFlush) {
|
||||
try {
|
||||
await params.flushPendingToolResultsAfterIdle({
|
||||
agent: params.session?.agent as IdleAwareAgent | null | undefined,
|
||||
sessionManager: params.sessionManager as ToolResultFlushManager | null | undefined,
|
||||
...(params.aborted ? { timeoutMs: 0 } : {}),
|
||||
});
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
// Release the write-lock before disposing runtimes so another attempt can
|
||||
// recover even if runtime disposal stalls or throws.
|
||||
await params.sessionLock.release();
|
||||
} catch (err) {
|
||||
sessionLockReleaseError = err;
|
||||
}
|
||||
params.removeToolResultContextGuard?.();
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
if (params.aborted && params.abortSettlePromise) {
|
||||
await waitForEmbeddedAbortSettle({
|
||||
promise: params.abortSettlePromise,
|
||||
runId: params.runId ?? "unknown",
|
||||
sessionId: params.sessionId ?? "unknown",
|
||||
});
|
||||
}
|
||||
try {
|
||||
await params.flushPendingToolResultsAfterIdle({
|
||||
agent: params.session?.agent as IdleAwareAgent | null | undefined,
|
||||
sessionManager: params.sessionManager as ToolResultFlushManager | null | undefined,
|
||||
...(params.aborted ? { timeoutMs: 0 } : {}),
|
||||
});
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -141,8 +120,4 @@ export async function cleanupEmbeddedAttemptResources(params: {
|
||||
} catch {
|
||||
/* best-effort */
|
||||
}
|
||||
|
||||
if (sessionLockReleaseError) {
|
||||
throw toErrorObject(sessionLockReleaseError, "Non-Error thrown");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,51 +53,6 @@ describe("runEmbeddedAttempt tool-search catalog cleanup", () => {
|
||||
tempPaths.length = 0;
|
||||
});
|
||||
|
||||
it("clears the registered run catalog when the run aborts during prep", async () => {
|
||||
const abortController = new AbortController();
|
||||
const prompt = vi.fn(async () => {});
|
||||
const abortError = new Error("stopped during lock acquisition");
|
||||
abortError.name = "AbortError";
|
||||
let markLockRequested!: () => void;
|
||||
const lockRequested = new Promise<void>((resolve) => {
|
||||
markLockRequested = resolve;
|
||||
});
|
||||
hoisted.acquireSessionWriteLockMock.mockImplementationOnce(async (params) => {
|
||||
markLockRequested();
|
||||
await new Promise<void>((resolve) => {
|
||||
params.signal?.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
throw params.signal?.reason;
|
||||
});
|
||||
hoisted.createOpenClawCodingToolsMock.mockImplementation(() => catalogProbeTools());
|
||||
|
||||
const runId = "run-catalog-abort";
|
||||
const attempt = createContextEngineAttemptRunner({
|
||||
contextEngine: createContextEngineBootstrapAndAssemble(),
|
||||
sessionKey: "agent:main:telegram:direct:123",
|
||||
tempPaths,
|
||||
sessionPrompt: prompt,
|
||||
attemptOverrides: {
|
||||
runId,
|
||||
abortSignal: abortController.signal,
|
||||
disableTools: false,
|
||||
config: {
|
||||
tools: { toolSearch: { enabled: true, mode: "tools" } },
|
||||
},
|
||||
},
|
||||
});
|
||||
await lockRequested;
|
||||
const catalogRef = requireAttemptCatalogRef();
|
||||
expect(catalogRef.current?.entries).toContainEqual(
|
||||
expect.objectContaining({ name: "cataloged_probe_tool" }),
|
||||
);
|
||||
|
||||
abortController.abort(abortError);
|
||||
await expect(attempt).rejects.toBe(abortError);
|
||||
|
||||
expect(catalogRef.current).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
mode: "code-mode",
|
||||
|
||||
@@ -31,7 +31,6 @@ import { prepareEmbeddedAttemptBundleTools } from "./attempt-bundle-tools.js";
|
||||
import { runEmbeddedAttemptExecutionPhase } from "./attempt-execution-phase.js";
|
||||
import type { EmbeddedAttemptExecutionState } from "./attempt-execution-types.js";
|
||||
import { cleanupEmbeddedAttemptSessionPhase } from "./attempt-session-cleanup.js";
|
||||
import { prepareEmbeddedAttemptSessionLock } from "./attempt-session-lock-prepare.js";
|
||||
import { prepareEmbeddedAttemptSessionRuntime } from "./attempt-session-runtime-prepare.js";
|
||||
import { prepareEmbeddedAttemptSetup } from "./attempt-setup.js";
|
||||
import { createEmbeddedRunStageTracker } from "./attempt-stage-timing.js";
|
||||
@@ -43,7 +42,7 @@ import {
|
||||
import { prepareEmbeddedAttemptSystemPrompt } from "./attempt-system-prompt-prepare.js";
|
||||
import { prepareEmbeddedAttemptToolBase } from "./attempt-tool-base-prepare.js";
|
||||
import { prepareEmbeddedAttemptToolCatalog } from "./attempt-tool-catalog.js";
|
||||
import type { EmbeddedAttemptSessionFileOwner } from "./attempt.session-lock.js";
|
||||
import { prepareEmbeddedAttemptTranscriptLifecycle } from "./attempt-transcript-lifecycle-prepare.js";
|
||||
import {
|
||||
queueSessionsYieldInterruptMessage,
|
||||
SESSIONS_YIELD_ABORT_REASON,
|
||||
@@ -97,9 +96,6 @@ export async function runEmbeddedAttempt(
|
||||
executionState.terminal = mergeAgentRunAttemptTerminal(executionState.terminal, incoming);
|
||||
};
|
||||
let emitDiagnosticRunCompleted: EmitDiagnosticRunCompleted | undefined;
|
||||
// Releases the eager session lock if post-prompt code exits before cleanup.
|
||||
let releaseRetainedSessionLock: (() => Promise<void>) | undefined;
|
||||
let retainedSessionFileOwner: EmbeddedAttemptSessionFileOwner | undefined;
|
||||
let bundleMcpRuntime: Awaited<ReturnType<typeof materializeBundleMcpToolsForRun>> | undefined;
|
||||
let bundleLspRuntime: Awaited<ReturnType<typeof createBundleLspToolRuntime>> | undefined;
|
||||
let toolSearchCatalogRef: ToolSearchCatalogRef | undefined;
|
||||
@@ -348,21 +344,14 @@ export async function runEmbeddedAttempt(
|
||||
const {
|
||||
compactionTimeoutMs,
|
||||
ownedTranscriptWriteContext,
|
||||
sessionLockController,
|
||||
withOwnedSessionWriteLock,
|
||||
transcriptLifecycle,
|
||||
withOwnedTranscriptWrite,
|
||||
} = await measureEmbeddedAgentPreparation(
|
||||
"attempt.session-lock",
|
||||
"attempt.transcript-lifecycle",
|
||||
() =>
|
||||
prepareEmbeddedAttemptSessionLock({
|
||||
prepareEmbeddedAttemptTranscriptLifecycle({
|
||||
attempt: params,
|
||||
externalAbortController,
|
||||
getSessionManager: () => sessionManager,
|
||||
onSessionFileOwnerAcquired: (owner) => {
|
||||
retainedSessionFileOwner = owner;
|
||||
},
|
||||
onSessionLockReleaseReady: (release) => {
|
||||
releaseRetainedSessionLock = release;
|
||||
},
|
||||
}),
|
||||
{ config: params.config },
|
||||
);
|
||||
@@ -390,8 +379,8 @@ export async function runEmbeddedAttempt(
|
||||
replayAllowedToolNames: toolSearchRunPlan.replayAllowedToolNames,
|
||||
resolveActiveContextEnginePluginId,
|
||||
sessionAgentId,
|
||||
sessionLockController,
|
||||
withOwnedSessionWriteLock,
|
||||
transcriptLifecycle,
|
||||
withOwnedTranscriptWrite,
|
||||
},
|
||||
agentSession: {
|
||||
agentCoreThinkingLevel,
|
||||
@@ -478,8 +467,7 @@ export async function runEmbeddedAttempt(
|
||||
sessionLock: {
|
||||
compactionTimeoutMs,
|
||||
ownedTranscriptWriteContext,
|
||||
sessionLockController,
|
||||
withOwnedSessionWriteLock,
|
||||
withOwnedTranscriptWrite,
|
||||
},
|
||||
setup: {
|
||||
effectiveFsWorkspaceOnly,
|
||||
@@ -521,7 +509,7 @@ export async function runEmbeddedAttempt(
|
||||
attempt: params,
|
||||
session,
|
||||
sessionManager,
|
||||
sessionLockController,
|
||||
transcriptLifecycle,
|
||||
bundleMcpRuntime,
|
||||
bundleLspRuntime,
|
||||
removeToolResultContextGuard,
|
||||
@@ -559,14 +547,6 @@ export async function runEmbeddedAttempt(
|
||||
`failed to clean up embedded prep resources after early attempt exit: runId=${params.runId} ${String(cleanupErr)}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
await releaseRetainedSessionLock?.();
|
||||
} catch (releaseErr) {
|
||||
log.error(
|
||||
`failed to release retained session lock on attempt teardown: runId=${params.runId} ${String(releaseErr)}`,
|
||||
);
|
||||
}
|
||||
retainedSessionFileOwner?.release();
|
||||
const terminal = projectAgentRunAttemptTerminal(executionState.terminal);
|
||||
emitDiagnosticRunCompleted?.(
|
||||
terminal.aborted ? "aborted" : "error",
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
} from "../../../config/sessions/session-accessor.js";
|
||||
import * as sessionAccessor from "../../../config/sessions/session-accessor.js";
|
||||
import { useTempSessionsFixture } from "../../../config/sessions/test-helpers.js";
|
||||
import { SessionTranscriptWriterClaimReboundError } from "../../../config/sessions/transcript-write-context.js";
|
||||
import { appendExactAssistantMessageToSessionTranscript } from "../../../config/sessions/transcript.js";
|
||||
import type { InternalSessionEntry } from "../../../config/sessions/types.js";
|
||||
import {
|
||||
@@ -166,7 +167,7 @@ describe("embedded run durable writer admission", () => {
|
||||
usage: buildUsageWithNoCost({}),
|
||||
}),
|
||||
),
|
||||
).toThrow("Session transcript header was not persisted");
|
||||
).toThrow(SessionTranscriptWriterClaimReboundError);
|
||||
|
||||
const staleAppend = await appendExactAssistantMessageToSessionTranscript({
|
||||
agentId: "main",
|
||||
|
||||
@@ -769,8 +769,8 @@ export function getActiveEmbeddedRunSnapshot(
|
||||
/**
|
||||
* Wait for active embedded runs to drain.
|
||||
*
|
||||
* Used during restarts so in-flight runs can release session write locks before
|
||||
* the next lifecycle starts. If no timeout is passed, waits indefinitely.
|
||||
* Used during restarts so in-flight runs can finish transcript writes before the
|
||||
* next lifecycle starts. If no timeout is passed, waits indefinitely.
|
||||
*/
|
||||
export async function waitForActiveEmbeddedRuns(
|
||||
timeoutMs?: number,
|
||||
|
||||
@@ -22,7 +22,6 @@ import {
|
||||
resolveModelFallbackError,
|
||||
} from "./failover-error.js";
|
||||
import { AgentHarnessSessionSupersededError } from "./harness/errors.js";
|
||||
import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js";
|
||||
|
||||
// OpenAI 429 example shape: https://help.openai.com/en/articles/5955604-how-can-i-solve-429-too-many-requests-errors
|
||||
const OPENAI_RATE_LIMIT_MESSAGE =
|
||||
@@ -528,87 +527,6 @@ describe("failover-error", () => {
|
||||
).toBe("overloaded");
|
||||
});
|
||||
|
||||
it("does not classify session lock wait errors as model timeout failover", () => {
|
||||
const sessionLockError = new SessionWriteLockTimeoutError({
|
||||
timeoutMs: 10_000,
|
||||
owner: "pid=37121",
|
||||
lockPath: "/tmp/openclaw/session.jsonl.lock",
|
||||
});
|
||||
expect(resolveFailoverReasonFromError(sessionLockError)).toBeNull();
|
||||
expect(isTimeoutError(sessionLockError)).toBe(false);
|
||||
|
||||
const wrappedLockError = Object.assign(new Error("operation timed out"), {
|
||||
name: "AbortError",
|
||||
cause: sessionLockError,
|
||||
});
|
||||
expect(resolveFailoverReasonFromError(wrappedLockError)).toBeNull();
|
||||
expect(isTimeoutError(wrappedLockError)).toBe(false);
|
||||
|
||||
const abortWrappedLockError = Object.assign(new Error("request was aborted"), {
|
||||
name: "AbortError",
|
||||
cause: sessionLockError,
|
||||
});
|
||||
expect(resolveFailoverReasonFromError(abortWrappedLockError)).toBeNull();
|
||||
expect(isTimeoutError(abortWrappedLockError)).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps explicit provider failover metadata authoritative over nested session lock text", () => {
|
||||
expect(
|
||||
resolveFailoverReasonFromError({
|
||||
status: 429,
|
||||
code: "RESOURCE_EXHAUSTED",
|
||||
message: "upstream quota pressure",
|
||||
cause: new SessionWriteLockTimeoutError({
|
||||
timeoutMs: 10_000,
|
||||
owner: "pid=37121",
|
||||
lockPath: "/tmp/openclaw/session.jsonl.lock",
|
||||
}),
|
||||
}),
|
||||
).toBe("rate_limit");
|
||||
});
|
||||
|
||||
it("keeps inferred HTTP failover metadata authoritative over nested session lock text", () => {
|
||||
expect(
|
||||
resolveFailoverReasonFromError({
|
||||
message: "HTTP 429: upstream quota pressure",
|
||||
cause: new SessionWriteLockTimeoutError({
|
||||
timeoutMs: 10_000,
|
||||
owner: "pid=37121",
|
||||
lockPath: "/tmp/openclaw/session.jsonl.lock",
|
||||
}),
|
||||
}),
|
||||
).toBe("rate_limit");
|
||||
});
|
||||
|
||||
it("does not treat generic abort codes as explicit failover metadata over nested session lock text", () => {
|
||||
expect(
|
||||
resolveFailoverReasonFromError({
|
||||
name: "AbortError",
|
||||
code: "ABORT_ERR",
|
||||
message: "The operation was aborted",
|
||||
cause: new SessionWriteLockTimeoutError({
|
||||
timeoutMs: 10_000,
|
||||
owner: "pid=37121",
|
||||
lockPath: "/tmp/openclaw/session.jsonl.lock",
|
||||
}),
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("does not let cause-based failover classification bypass wrapper session lock suppression", () => {
|
||||
expect(
|
||||
resolveFailoverReasonFromError({
|
||||
message: "wrapper",
|
||||
reason: new SessionWriteLockTimeoutError({
|
||||
timeoutMs: 10_000,
|
||||
owner: "pid=37121",
|
||||
lockPath: "/tmp/openclaw/session.jsonl.lock",
|
||||
}),
|
||||
cause: new Error("operation timed out"),
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("classifies bare shared model runtime stream wrapper as timeout regardless of provider (#71620)", () => {
|
||||
expect(
|
||||
resolveFailoverReasonFromError({
|
||||
@@ -1411,26 +1329,14 @@ describe("failover-error", () => {
|
||||
});
|
||||
|
||||
describe("isNonProviderRuntimeCoordinationError", () => {
|
||||
const makeSessionLockError = () =>
|
||||
new SessionWriteLockTimeoutError({
|
||||
timeoutMs: 10_000,
|
||||
owner: "pid=37121",
|
||||
lockPath: "/tmp/openclaw/session.jsonl.lock",
|
||||
});
|
||||
const makeEmbeddedTakeoverError = () => {
|
||||
const err = new Error(
|
||||
"session file changed while embedded prompt lock was released: /tmp/openclaw/session.jsonl",
|
||||
);
|
||||
err.name = "EmbeddedAttemptSessionTakeoverError";
|
||||
const makeWriterClaimReboundError = () => {
|
||||
const err = new Error("session writer claim changed before transcript persistence");
|
||||
err.name = "SessionTranscriptWriterClaimReboundError";
|
||||
return err;
|
||||
};
|
||||
|
||||
it("returns true for direct session write-lock timeout errors", () => {
|
||||
expect(isNonProviderRuntimeCoordinationError(makeSessionLockError())).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for direct embedded attempt session takeover errors", () => {
|
||||
expect(isNonProviderRuntimeCoordinationError(makeEmbeddedTakeoverError())).toBe(true);
|
||||
it("returns true for transcript writer claim rebound errors", () => {
|
||||
expect(isNonProviderRuntimeCoordinationError(makeWriterClaimReboundError())).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for harness session generation ownership loss", () => {
|
||||
@@ -1467,11 +1373,8 @@ describe("failover-error", () => {
|
||||
});
|
||||
|
||||
it("returns true when the coordination error is nested via cause", () => {
|
||||
const wrapped = new Error("wrapper", { cause: makeSessionLockError() });
|
||||
expect(isNonProviderRuntimeCoordinationError(wrapped)).toBe(true);
|
||||
|
||||
const wrappedTakeover = new Error("wrapper", { cause: makeEmbeddedTakeoverError() });
|
||||
expect(isNonProviderRuntimeCoordinationError(wrappedTakeover)).toBe(true);
|
||||
const wrappedRebound = new Error("wrapper", { cause: makeWriterClaimReboundError() });
|
||||
expect(isNonProviderRuntimeCoordinationError(wrappedRebound)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for Codex missing tool-result local execution failures", () => {
|
||||
@@ -1494,14 +1397,6 @@ describe("failover-error", () => {
|
||||
expect(isNonProviderRuntimeCoordinationError({ status: 429, message: "rate limit" })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
isNonProviderRuntimeCoordinationError({
|
||||
status: 429,
|
||||
code: "RESOURCE_EXHAUSTED",
|
||||
message: "upstream quota pressure",
|
||||
cause: makeSessionLockError(),
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isNonProviderRuntimeCoordinationError({
|
||||
status: 503,
|
||||
@@ -1529,63 +1424,12 @@ describe("failover-error", () => {
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when takeover wrapper holds a classifiable provider timeout in promptError", () => {
|
||||
// Simulates EmbeddedAttemptPromptErrorWithCleanupTakeoverError: name matches
|
||||
// EmbeddedAttemptSessionTakeoverError, but the real cause is in .promptError.
|
||||
const timeoutPromptErr = Object.assign(new Error("request timed out"), {
|
||||
name: "TimeoutError",
|
||||
});
|
||||
const wrapper = Object.assign(new Error("request timed out"), {
|
||||
name: "EmbeddedAttemptSessionTakeoverError",
|
||||
promptError: timeoutPromptErr,
|
||||
});
|
||||
expect(isNonProviderRuntimeCoordinationError(wrapper)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when takeover wrapper holds rate_limit in promptError", () => {
|
||||
const rateLimitPromptErr = {
|
||||
status: 429,
|
||||
code: "RATE_LIMITED",
|
||||
message: "too many requests",
|
||||
};
|
||||
const wrapper = Object.assign(new Error("cleanup takeover"), {
|
||||
name: "EmbeddedAttemptSessionTakeoverError",
|
||||
promptError: rateLimitPromptErr,
|
||||
});
|
||||
expect(isNonProviderRuntimeCoordinationError(wrapper)).toBe(false);
|
||||
const resolution = resolveModelFallbackError(wrapper);
|
||||
expect(resolution).toMatchObject({
|
||||
kind: "failover",
|
||||
error: {
|
||||
message: "too many requests",
|
||||
reason: "rate_limit",
|
||||
status: 429,
|
||||
code: "RATE_LIMITED",
|
||||
},
|
||||
});
|
||||
expect(resolution.error).toHaveProperty("cause", wrapper);
|
||||
});
|
||||
|
||||
it("returns true when takeover wrapper holds an unclassifiable promptError", () => {
|
||||
const unknownPromptErr = { weirdField: "something unknown" };
|
||||
const wrapper = Object.assign(new Error("unknown"), {
|
||||
name: "EmbeddedAttemptSessionTakeoverError",
|
||||
promptError: unknownPromptErr,
|
||||
});
|
||||
expect(isNonProviderRuntimeCoordinationError(wrapper)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for pure takeover error without promptError (regression)", () => {
|
||||
const pureTakeover = Object.assign(
|
||||
new Error("session file changed while embedded prompt lock was released"),
|
||||
{ name: "EmbeddedAttemptSessionTakeoverError" },
|
||||
);
|
||||
expect(isNonProviderRuntimeCoordinationError(pureTakeover)).toBe(true);
|
||||
it("keeps provider-looking rebound wrappers in the coordination path", () => {
|
||||
const rebound = makeWriterClaimReboundError();
|
||||
expect(isNonProviderRuntimeCoordinationError(rebound)).toBe(true);
|
||||
expect(
|
||||
isNonProviderRuntimeCoordinationError(
|
||||
Object.assign(new Error("provider rejected request: rate limit"), {
|
||||
name: "EmbeddedAttemptSessionTakeoverError",
|
||||
}),
|
||||
new Error("provider rejected request: rate limit", { cause: rebound }),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
@@ -10,7 +10,6 @@ import { collectErrorGraphCandidates, readErrorName } from "../infra/errors.js";
|
||||
import {
|
||||
classifyFailoverSignal,
|
||||
extractFailoverSignalDetails,
|
||||
inferSignalStatus,
|
||||
isUnclassifiedNoBodyHttpSignal,
|
||||
type FailoverClassification,
|
||||
type FailoverSignal,
|
||||
@@ -18,7 +17,6 @@ import {
|
||||
import { isTimeoutErrorMessage } from "./embedded-agent-helpers/errors.js";
|
||||
import type { FailoverReason } from "./embedded-agent-helpers/types.js";
|
||||
import { AgentHarnessSessionSupersededError } from "./harness/errors.js";
|
||||
import { isSessionWriteLockAcquireError } from "./session-write-lock-error.js";
|
||||
|
||||
const ABORT_TIMEOUT_RE = /request was aborted|request aborted/i;
|
||||
const MAX_FAILOVER_CAUSE_DEPTH = 25;
|
||||
@@ -390,49 +388,15 @@ function normalizeDirectErrorSignal(err: unknown): FailoverSignal {
|
||||
};
|
||||
}
|
||||
|
||||
function hasSessionWriteLockContention(err: unknown, seen: Set<object> = new Set()): boolean {
|
||||
if (isSessionWriteLockAcquireError(err)) {
|
||||
return true;
|
||||
}
|
||||
if (!err || typeof err !== "object") {
|
||||
return false;
|
||||
}
|
||||
if (seen.has(err)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(err);
|
||||
const candidate = err as { error?: unknown; cause?: unknown; reason?: unknown };
|
||||
return (
|
||||
hasSessionWriteLockContention(candidate.error, seen) ||
|
||||
hasSessionWriteLockContention(candidate.cause, seen) ||
|
||||
hasSessionWriteLockContention(candidate.reason, seen)
|
||||
);
|
||||
}
|
||||
|
||||
function isEmbeddedAttemptSessionTakeover(err: unknown): boolean {
|
||||
// Match by name to avoid importing embedded-agent-runner here (would create a cycle).
|
||||
return Boolean(
|
||||
err && typeof err === "object" && readErrorName(err) === "EmbeddedAttemptSessionTakeoverError",
|
||||
);
|
||||
}
|
||||
|
||||
function hasPreservedTakeoverPromptError(err: unknown): err is Record<"promptError", unknown> {
|
||||
return Boolean(
|
||||
isEmbeddedAttemptSessionTakeover(err) &&
|
||||
function hasSessionTranscriptWriterClaimRebound(
|
||||
err: unknown,
|
||||
seen: Set<object> = new Set(),
|
||||
): boolean {
|
||||
if (
|
||||
err &&
|
||||
typeof err === "object" &&
|
||||
Object.hasOwn(err, "promptError"),
|
||||
);
|
||||
}
|
||||
|
||||
function resolveFailoverSourceError(err: unknown): unknown {
|
||||
// Cleanup takeover is a secondary failure when the wrapper preserves the
|
||||
// prompt error. Classify and report that provider-facing source instead.
|
||||
return hasPreservedTakeoverPromptError(err) ? err.promptError : err;
|
||||
}
|
||||
|
||||
function hasEmbeddedAttemptSessionTakeover(err: unknown, seen: Set<object> = new Set()): boolean {
|
||||
if (isEmbeddedAttemptSessionTakeover(err)) {
|
||||
readErrorName(err) === "SessionTranscriptWriterClaimReboundError"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (!err || typeof err !== "object") {
|
||||
@@ -444,9 +408,9 @@ function hasEmbeddedAttemptSessionTakeover(err: unknown, seen: Set<object> = new
|
||||
seen.add(err);
|
||||
const candidate = err as { error?: unknown; cause?: unknown; reason?: unknown };
|
||||
return (
|
||||
hasEmbeddedAttemptSessionTakeover(candidate.error, seen) ||
|
||||
hasEmbeddedAttemptSessionTakeover(candidate.cause, seen) ||
|
||||
hasEmbeddedAttemptSessionTakeover(candidate.reason, seen)
|
||||
hasSessionTranscriptWriterClaimRebound(candidate.error, seen) ||
|
||||
hasSessionTranscriptWriterClaimRebound(candidate.cause, seen) ||
|
||||
hasSessionTranscriptWriterClaimRebound(candidate.reason, seen)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -534,9 +498,6 @@ function hasTimeoutHint(err: unknown): boolean {
|
||||
if (!err) {
|
||||
return false;
|
||||
}
|
||||
if (hasSessionWriteLockContention(err)) {
|
||||
return false;
|
||||
}
|
||||
if (readErrorName(err) === "TimeoutError") {
|
||||
return true;
|
||||
}
|
||||
@@ -555,9 +516,6 @@ export function isTimeoutError(err: unknown): boolean {
|
||||
if (readErrorName(err) !== "AbortError") {
|
||||
return false;
|
||||
}
|
||||
if (hasSessionWriteLockContention(err)) {
|
||||
return false;
|
||||
}
|
||||
const message = getErrorMessage(err);
|
||||
if (message && ABORT_TIMEOUT_RE.test(message)) {
|
||||
return true;
|
||||
@@ -667,14 +625,6 @@ function resolveFailoverClassificationFromErrorInternal(
|
||||
};
|
||||
}
|
||||
const signal = normalizeErrorSignal(err, providerHint);
|
||||
const codeReason = signal.code
|
||||
? failoverReasonFromClassification(classifyFailoverSignal({ code: signal.code }))
|
||||
: null;
|
||||
const hasExplicitFailoverMetadata =
|
||||
typeof inferSignalStatus(signal) === "number" ||
|
||||
(codeReason !== null && codeReason !== "timeout");
|
||||
const hasSessionLock = hasSessionWriteLockContention(err);
|
||||
|
||||
const classification = classifyFailoverSignal(signal);
|
||||
const nestedCandidates = getNestedErrorCandidates(err);
|
||||
|
||||
@@ -687,9 +637,6 @@ function resolveFailoverClassificationFromErrorInternal(
|
||||
providerHint,
|
||||
);
|
||||
if (nestedClassification) {
|
||||
if (hasSessionLock && !hasExplicitFailoverMetadata) {
|
||||
return null;
|
||||
}
|
||||
return nestedClassification;
|
||||
}
|
||||
}
|
||||
@@ -713,16 +660,9 @@ function resolveFailoverClassificationFromErrorInternal(
|
||||
}
|
||||
|
||||
if (classification) {
|
||||
if (hasSessionLock && !hasExplicitFailoverMetadata) {
|
||||
return null;
|
||||
}
|
||||
return classification;
|
||||
}
|
||||
|
||||
if (hasSessionLock) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (isTimeoutError(err)) {
|
||||
return {
|
||||
kind: "reason",
|
||||
@@ -865,7 +805,7 @@ export function coerceToFailoverError(
|
||||
err: unknown,
|
||||
context?: FailoverErrorContext,
|
||||
): FailoverError | null {
|
||||
const sourceError = resolveFailoverSourceError(err);
|
||||
const sourceError = err;
|
||||
if (isFailoverError(sourceError)) {
|
||||
if (context?.authMode && !sourceError.authMode) {
|
||||
const message =
|
||||
@@ -938,22 +878,16 @@ export function resolveModelFallbackError(
|
||||
) {
|
||||
return { kind: "coordination", error: err };
|
||||
}
|
||||
// A direct takeover remains a coordination failure unless the dedicated
|
||||
// cleanup wrapper owns a preserved prompt error. Its message alone must not
|
||||
// reclassify session-state loss as a provider failure.
|
||||
if (isEmbeddedAttemptSessionTakeover(err) && !hasPreservedTakeoverPromptError(err)) {
|
||||
// The in-transaction transcript fence owns writer supersession. A rebound is
|
||||
// local coordination failure even when provider-looking wrappers contain it.
|
||||
if (hasSessionTranscriptWriterClaimRebound(err)) {
|
||||
return { kind: "coordination", error: err };
|
||||
}
|
||||
const failoverError = coerceToFailoverError(err, context);
|
||||
if (failoverError) {
|
||||
return { kind: "failover", error: failoverError };
|
||||
}
|
||||
if (
|
||||
hasSessionWriteLockContention(err) ||
|
||||
hasEmbeddedAttemptSessionTakeover(err) ||
|
||||
hasMissingToolResultFailure(err) ||
|
||||
staleLifecycleFailure
|
||||
) {
|
||||
if (hasMissingToolResultFailure(err) || staleLifecycleFailure) {
|
||||
return { kind: "coordination", error: err };
|
||||
}
|
||||
return { kind: "unknown", error: err };
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { runWithSessionTranscriptReadFence } from "../../config/sessions/session-transcript-read-fence.js";
|
||||
/**
|
||||
* Manages context-engine lifecycle hooks for native agent harnesses.
|
||||
@@ -25,7 +26,6 @@ import {
|
||||
} from "../embedded-agent-runner/run/attempt.prompt-helpers.js";
|
||||
import { stripRuntimeContextCustomMessages } from "../internal-runtime-context.js";
|
||||
import type { AgentMessage } from "../runtime/index.js";
|
||||
import type { SessionWriteLockAcquireTimeoutConfig } from "../session-write-lock.js";
|
||||
|
||||
type HarnessContextEngine = ContextEngine;
|
||||
|
||||
@@ -116,7 +116,7 @@ export async function bootstrapHarnessContextEngine(params: {
|
||||
fallbackReason?: string | null;
|
||||
degradedReason?: string | null;
|
||||
runMaintenance?: typeof runHarnessContextEngineMaintenance;
|
||||
config?: SessionWriteLockAcquireTimeoutConfig;
|
||||
config?: OpenClawConfig;
|
||||
warn: (message: string) => void;
|
||||
}): Promise<void> {
|
||||
if (
|
||||
@@ -286,7 +286,7 @@ export async function finalizeHarnessContextEngineTurn(params: {
|
||||
degradedReason?: string | null;
|
||||
runMaintenance?: typeof runHarnessContextEngineMaintenance;
|
||||
sessionManager?: unknown;
|
||||
config?: SessionWriteLockAcquireTimeoutConfig;
|
||||
config?: OpenClawConfig;
|
||||
warn: (message: string) => void;
|
||||
/** True when this turn belongs to a heartbeat run. */
|
||||
isHeartbeat?: boolean;
|
||||
@@ -442,7 +442,7 @@ export async function runHarnessContextEngineMaintenance(params: {
|
||||
executionMode?: "foreground" | "background";
|
||||
onDeferredMaintenance?: (promise: Promise<void>) => void;
|
||||
withSessionManagerRewriteLock?: <T>(operation: () => Promise<T> | T) => Promise<T>;
|
||||
config?: SessionWriteLockAcquireTimeoutConfig;
|
||||
config?: OpenClawConfig;
|
||||
}) {
|
||||
const runtimeSettings = buildHarnessContextEngineRuntimeSettings(params);
|
||||
return await runContextEngineMaintenance({
|
||||
|
||||
@@ -45,7 +45,6 @@ import {
|
||||
} from "./run-termination.js";
|
||||
import { toSandboxProvisioningError } from "./sandbox/provisioning-error.js";
|
||||
import { resolveSessionSuspensionReason } from "./session-suspension.js";
|
||||
import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js";
|
||||
import { makeModelFallbackCfg } from "./test-helpers/model-fallback-config-fixture.js";
|
||||
|
||||
const emptyManifestPlugins = [] as const;
|
||||
@@ -1968,7 +1967,7 @@ describe("runWithModelFallback", () => {
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("aborts the fallback chain on embedded session takeover instead of trying every model (#83510)", async () => {
|
||||
it("aborts the fallback chain when the transcript writer claim rebounds", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
@@ -1979,11 +1978,9 @@ describe("runWithModelFallback", () => {
|
||||
},
|
||||
},
|
||||
});
|
||||
const takeoverError = new Error(
|
||||
"session file changed while embedded prompt lock was released: /tmp/session.jsonl",
|
||||
);
|
||||
takeoverError.name = "EmbeddedAttemptSessionTakeoverError";
|
||||
const run = vi.fn().mockRejectedValue(takeoverError);
|
||||
const reboundError = new Error("session writer claim changed before transcript persistence");
|
||||
reboundError.name = "SessionTranscriptWriterClaimReboundError";
|
||||
const run = vi.fn().mockRejectedValue(reboundError);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
@@ -1992,7 +1989,7 @@ describe("runWithModelFallback", () => {
|
||||
model: "gpt-5.4",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(takeoverError);
|
||||
).rejects.toBe(reboundError);
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -2184,7 +2181,7 @@ describe("runWithModelFallback", () => {
|
||||
expect(onFallbackStep).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("aborts fallback when a provider prompt error carries cleanup session takeover", async () => {
|
||||
it("aborts fallback when a provider-looking wrapper carries writer claim rebound", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
@@ -2195,14 +2192,13 @@ describe("runWithModelFallback", () => {
|
||||
},
|
||||
},
|
||||
});
|
||||
const cleanupTakeover = new Error(
|
||||
"session file changed while embedded prompt lock was released: /tmp/session.jsonl",
|
||||
const writerClaimRebound = new Error(
|
||||
"session writer claim changed before transcript persistence",
|
||||
);
|
||||
cleanupTakeover.name = "EmbeddedAttemptSessionTakeoverError";
|
||||
writerClaimRebound.name = "SessionTranscriptWriterClaimReboundError";
|
||||
const providerFacingError = new Error("provider rejected request: rate limit", {
|
||||
cause: cleanupTakeover,
|
||||
cause: writerClaimRebound,
|
||||
});
|
||||
providerFacingError.name = "EmbeddedAttemptSessionTakeoverError";
|
||||
const run = vi.fn().mockRejectedValue(providerFacingError);
|
||||
|
||||
await expect(
|
||||
@@ -2216,35 +2212,6 @@ describe("runWithModelFallback", () => {
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("aborts the fallback chain on session write-lock timeout instead of trying every model (#83510)", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "openai/gpt-5.4",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-6", "openai/gpt-4.1-mini"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const lockError = new SessionWriteLockTimeoutError({
|
||||
timeoutMs: 10_000,
|
||||
owner: "pid=37121",
|
||||
lockPath: "/tmp/openclaw/session.jsonl.lock",
|
||||
});
|
||||
const run = vi.fn().mockRejectedValue(lockError);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(lockError);
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("aborts the fallback chain on stale gateway lifecycle errors (#116418)", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
@@ -2423,49 +2390,6 @@ describe("runWithModelFallback", () => {
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps provider failover metadata authoritative over nested session locks", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "openai/gpt-5.4",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-6"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const lockError = new SessionWriteLockTimeoutError({
|
||||
timeoutMs: 10_000,
|
||||
owner: "pid=37121",
|
||||
lockPath: "/tmp/openclaw/session.jsonl.lock",
|
||||
});
|
||||
const providerError = {
|
||||
status: 429,
|
||||
code: "RESOURCE_EXHAUSTED",
|
||||
message: "upstream quota pressure",
|
||||
cause: lockError,
|
||||
};
|
||||
const run = vi.fn().mockRejectedValueOnce(providerError).mockResolvedValueOnce("fallback ok");
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
run,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("fallback ok");
|
||||
expect(result.provider).toBe("anthropic");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(result.attempts[0]).toMatchObject({
|
||||
provider: "openai",
|
||||
model: "gpt-5.4",
|
||||
reason: "rate_limit",
|
||||
status: 429,
|
||||
code: "RESOURCE_EXHAUSTED",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps raw provider schema errors in fallback summaries", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
@@ -5358,90 +5282,4 @@ describe("runWithImageModelFallback", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("runWithModelFallback preserved prompt errors", () => {
|
||||
it.each([
|
||||
{
|
||||
label: "timeout",
|
||||
promptError: Object.assign(new Error("request timed out"), { name: "TimeoutError" }),
|
||||
expected: { message: "request timed out", reason: "timeout", status: 408 },
|
||||
},
|
||||
{
|
||||
label: "rate limit",
|
||||
promptError: { status: 429, code: "RATE_LIMITED", message: "too many requests" },
|
||||
expected: {
|
||||
message: "too many requests",
|
||||
reason: "rate_limit",
|
||||
status: 429,
|
||||
code: "RATE_LIMITED",
|
||||
},
|
||||
},
|
||||
])(
|
||||
"falls back with the preserved $label prompt error (#99963)",
|
||||
async ({ promptError, expected }) => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "openai/gpt-5.5",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-6"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const takeoverError = Object.assign(new Error("cleanup takeover"), {
|
||||
name: "EmbeddedAttemptSessionTakeoverError",
|
||||
promptError,
|
||||
});
|
||||
const run = vi.fn().mockRejectedValueOnce(takeoverError).mockResolvedValueOnce("ok");
|
||||
const onError = vi.fn();
|
||||
|
||||
const result = await runWithModelFallback({
|
||||
cfg,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
run,
|
||||
onError,
|
||||
});
|
||||
|
||||
expect(result.result).toBe("ok");
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(result.attempts[0]).toMatchObject({
|
||||
error: expected.message,
|
||||
reason: expected.reason,
|
||||
});
|
||||
const observedError = onError.mock.calls[0]?.[0]?.error;
|
||||
expect(observedError).toMatchObject({ name: "FailoverError", ...expected });
|
||||
expect(observedError).toHaveProperty("cause", takeoverError);
|
||||
},
|
||||
);
|
||||
|
||||
it("still aborts fallback for a pure takeover error without promptError (regression)", async () => {
|
||||
const cfg = makeCfg({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: {
|
||||
primary: "openai/gpt-5.5",
|
||||
fallbacks: ["anthropic/claude-sonnet-4-6"],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const pureTakeoverError = Object.assign(
|
||||
new Error("session file changed while embedded prompt lock was released"),
|
||||
{ name: "EmbeddedAttemptSessionTakeoverError" },
|
||||
);
|
||||
const run = vi.fn().mockRejectedValue(pureTakeoverError);
|
||||
|
||||
await expect(
|
||||
runWithModelFallback({
|
||||
cfg,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
run,
|
||||
}),
|
||||
).rejects.toBe(pureTakeoverError);
|
||||
expect(run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
/** Stable session ownership errors shared by runtime and public harness adapters. */
|
||||
const TIMEOUT_CODE = "OPENCLAW_SESSION_WRITE_LOCK_TIMEOUT";
|
||||
const STALE_CODE = "OPENCLAW_SESSION_WRITE_LOCK_STALE";
|
||||
|
||||
export class SessionWriteLockTimeoutError extends Error {
|
||||
readonly code = TIMEOUT_CODE;
|
||||
declare readonly timeoutMs: number;
|
||||
declare readonly owner: string;
|
||||
declare readonly lockPath: string;
|
||||
|
||||
constructor(params: { timeoutMs: number; owner: string; lockPath: string }) {
|
||||
super(
|
||||
`session file locked (timeout ${params.timeoutMs}ms): ${params.owner} ${params.lockPath}`,
|
||||
);
|
||||
this.name = "SessionWriteLockTimeoutError";
|
||||
Object.assign(this, params);
|
||||
}
|
||||
}
|
||||
|
||||
export class SessionWriteLockStaleError extends Error {
|
||||
readonly code = STALE_CODE;
|
||||
declare readonly owner: string;
|
||||
declare readonly lockPath: string;
|
||||
declare readonly staleReasons: string[];
|
||||
|
||||
constructor(params: { owner: string; lockPath: string; staleReasons?: string[] }) {
|
||||
const staleReasons = params.staleReasons?.length ? params.staleReasons : ["unknown"];
|
||||
super(
|
||||
`session file lock stale (${staleReasons.join(", ")}): ${params.owner} ${params.lockPath}`,
|
||||
);
|
||||
this.name = "SessionWriteLockStaleError";
|
||||
Object.assign(this, params, { staleReasons });
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns whether another owner replaced the active SQLite session lease. */
|
||||
export function isSessionWriteLockLeaseLostError(error: unknown): boolean {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
const staleReasons = (error as { staleReasons?: unknown } | null)?.staleReasons;
|
||||
return (
|
||||
(error instanceof SessionWriteLockStaleError || code === STALE_CODE) &&
|
||||
Array.isArray(staleReasons) &&
|
||||
staleReasons.includes("lease-lost")
|
||||
);
|
||||
}
|
||||
|
||||
export function isSessionWriteLockAcquireError(error: unknown): boolean {
|
||||
const code = (error as { code?: unknown } | null)?.code;
|
||||
return (
|
||||
error instanceof SessionWriteLockTimeoutError ||
|
||||
error instanceof SessionWriteLockStaleError ||
|
||||
code === TIMEOUT_CODE ||
|
||||
code === STALE_CODE
|
||||
);
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
import "./session-write-lock.js";
|
||||
|
||||
type SessionWriteLockTestApi = {
|
||||
resetSessionWriteLockStateForTest(): void;
|
||||
testing: {
|
||||
releaseAllLocksSync(): void;
|
||||
setProcessStartTimeResolverForTest(resolver: ((pid: number) => number | null) | null): void;
|
||||
};
|
||||
};
|
||||
|
||||
function getTestApi(): SessionWriteLockTestApi {
|
||||
return (globalThis as Record<PropertyKey, unknown>)[
|
||||
Symbol.for("openclaw.sessionWriteLockTestApi")
|
||||
] as SessionWriteLockTestApi;
|
||||
}
|
||||
|
||||
export function resetSessionWriteLockStateForTest(): void {
|
||||
getTestApi().resetSessionWriteLockStateForTest();
|
||||
}
|
||||
|
||||
export const testing: SessionWriteLockTestApi["testing"] = {
|
||||
releaseAllLocksSync() {
|
||||
return getTestApi().testing.releaseAllLocksSync();
|
||||
},
|
||||
setProcessStartTimeResolverForTest(resolver) {
|
||||
return getTestApi().testing.setProcessStartTimeResolverForTest(resolver);
|
||||
},
|
||||
};
|
||||
@@ -1,287 +0,0 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
|
||||
import { acquireOpenClawStateLease } from "../state/openclaw-state-lease.js";
|
||||
import { SessionWriteLockTimeoutError } from "./session-write-lock-error.js";
|
||||
import {
|
||||
acquireSessionWriteLock,
|
||||
drainSessionWriteLockStateForTest,
|
||||
resolveSessionLockMaxHoldFromTimeout,
|
||||
resolveSessionWriteLockAcquireTimeoutMs,
|
||||
resolveSessionWriteLockOptions,
|
||||
resolveSessionWriteLockTargetKey,
|
||||
} from "./session-write-lock.js";
|
||||
import { resetSessionWriteLockStateForTest, testing } from "./session-write-lock.test-support.js";
|
||||
|
||||
describe("SQLite session write leases", () => {
|
||||
let root = "";
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-lease-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await drainSessionWriteLockStateForTest();
|
||||
resetSessionWriteLockStateForTest();
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function target(overrides: { agentId?: string; sessionId?: string; storePath?: string } = {}) {
|
||||
return resolveSessionWriteLockTargetKey({
|
||||
agentId: overrides.agentId ?? "main",
|
||||
sessionId: overrides.sessionId ?? "session-a",
|
||||
sessionKey: "agent:main:ignored-alias",
|
||||
storePath: overrides.storePath ?? path.join(root, "openclaw-agent.sqlite"),
|
||||
});
|
||||
}
|
||||
|
||||
async function acquire(
|
||||
sessionFile = target(),
|
||||
options: {
|
||||
timeoutMs?: number;
|
||||
maxHoldMs?: number;
|
||||
signal?: AbortSignal;
|
||||
allowReentrant?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
return await acquireSessionWriteLock({
|
||||
sessionFile,
|
||||
targetKind: "session-key",
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
it("serializes writers in SQLite without creating a sidecar", async () => {
|
||||
const sessionFile = target();
|
||||
const first = await acquire(sessionFile);
|
||||
|
||||
const timeout = await acquire(sessionFile, { timeoutMs: 5 }).catch((error: unknown) => error);
|
||||
expect(timeout).toBeInstanceOf(SessionWriteLockTimeoutError);
|
||||
expect(timeout).toMatchObject({
|
||||
owner: `held by this process (pid ${process.pid}); an earlier run has not released it`,
|
||||
});
|
||||
expect(String(timeout)).toContain(`pid ${process.pid}`);
|
||||
await expect(fs.access(`${sessionFile}.lock`)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
|
||||
await first.release();
|
||||
const second = await acquire(sessionFile, { timeoutMs: 500 });
|
||||
expect(() => second.assertOwned?.()).not.toThrow();
|
||||
await second.release();
|
||||
});
|
||||
|
||||
it("reports the live foreign process that holds the session lease", async () => {
|
||||
const databasePath = path.join(root, "foreign-openclaw-agent.sqlite");
|
||||
const sessionFile = target({ sessionId: "foreign-session", storePath: databasePath });
|
||||
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
await once(child, "spawn");
|
||||
const childPid = child.pid;
|
||||
if (!childPid) {
|
||||
throw new Error("foreign lease fixture did not start");
|
||||
}
|
||||
const foreignLease = await acquireOpenClawStateLease({
|
||||
scope: "session-write",
|
||||
key: sessionFile,
|
||||
database: { scope: "agent", agentId: "main", path: databasePath },
|
||||
leaseMs: 1_000,
|
||||
waitMs: 0,
|
||||
processOwner: {
|
||||
pid: childPid,
|
||||
startTime: null,
|
||||
isAlive: () => true,
|
||||
readStartTime: () => null,
|
||||
},
|
||||
});
|
||||
try {
|
||||
const timeout = await acquire(sessionFile, { timeoutMs: 5 }).catch((error: unknown) => error);
|
||||
expect(timeout).toBeInstanceOf(SessionWriteLockTimeoutError);
|
||||
expect(timeout).toMatchObject({ owner: `held by OpenClaw process pid ${childPid}` });
|
||||
} finally {
|
||||
await foreignLease.release();
|
||||
child.kill();
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
await once(child, "exit");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("reference-counts intentional process-local reentrancy", async () => {
|
||||
const sessionFile = target();
|
||||
const first = await acquire(sessionFile, { allowReentrant: true });
|
||||
const second = await acquire(sessionFile, { allowReentrant: true });
|
||||
|
||||
await first.release();
|
||||
expect(() => second.assertOwned?.()).not.toThrow();
|
||||
await expect(acquire(sessionFile, { timeoutMs: 5 })).rejects.toThrow(/session file locked/);
|
||||
await second.release();
|
||||
const next = await acquire(sessionFile, { timeoutMs: 500 });
|
||||
await next.release();
|
||||
});
|
||||
|
||||
it("coalesces repeated release and never erases a successor", async () => {
|
||||
const sessionFile = target();
|
||||
const first = await acquire(sessionFile);
|
||||
await Promise.all([first.release(), first.release()]);
|
||||
|
||||
const successor = await acquire(sessionFile, { timeoutMs: 500 });
|
||||
await first.release();
|
||||
expect(() => successor.assertOwned?.()).not.toThrow();
|
||||
await successor.release();
|
||||
});
|
||||
|
||||
it("uses abort only for admission", async () => {
|
||||
const alreadyAborted = new AbortController();
|
||||
const reason = new Error("stop requested");
|
||||
alreadyAborted.abort(reason);
|
||||
await expect(acquire(target(), { signal: alreadyAborted.signal })).rejects.toBe(reason);
|
||||
|
||||
const admittedAbort = new AbortController();
|
||||
const lock = await acquire(target({ sessionId: "session-b" }), {
|
||||
signal: admittedAbort.signal,
|
||||
});
|
||||
admittedAbort.abort(new Error("late abort"));
|
||||
expect(() => lock.assertOwned?.()).not.toThrow();
|
||||
await expect(lock.release()).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("aborts while waiting without affecting the current owner", async () => {
|
||||
const sessionFile = target();
|
||||
const held = await acquire(sessionFile);
|
||||
const abort = new AbortController();
|
||||
const reason = new Error("stop waiting");
|
||||
const pending = acquire(sessionFile, {
|
||||
timeoutMs: Number.POSITIVE_INFINITY,
|
||||
signal: abort.signal,
|
||||
});
|
||||
const rejected = expect(pending).rejects.toBe(reason);
|
||||
|
||||
abort.abort(reason);
|
||||
|
||||
await rejected;
|
||||
expect(() => held.assertOwned?.()).not.toThrow();
|
||||
await held.release();
|
||||
const successor = await acquire(sessionFile, { timeoutMs: 500 });
|
||||
await successor.release();
|
||||
});
|
||||
|
||||
it("does not time-take over a live lease owner", async () => {
|
||||
testing.setProcessStartTimeResolverForTest(() => null);
|
||||
const sessionFile = target();
|
||||
const first = await acquire(sessionFile, { maxHoldMs: 5 });
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, 10);
|
||||
});
|
||||
|
||||
await expect(acquire(sessionFile, { maxHoldMs: 1_000, timeoutMs: 5 })).rejects.toThrow(
|
||||
/session file locked/,
|
||||
);
|
||||
expect(() => first.assertOwned?.()).not.toThrow();
|
||||
await first.release();
|
||||
});
|
||||
|
||||
it("invalidates held handles during synchronous process cleanup", async () => {
|
||||
const lock = await acquire();
|
||||
testing.releaseAllLocksSync();
|
||||
expect(() => lock.assertOwned?.()).toThrow(/lease-lost/);
|
||||
await expect(lock.release()).resolves.toBeUndefined();
|
||||
const next = await acquire(target(), { timeoutMs: 500 });
|
||||
await next.release();
|
||||
});
|
||||
|
||||
it("namespaces independent transcript targets", async () => {
|
||||
const main = await acquire(target({ agentId: "main", sessionId: "main-session" }));
|
||||
const worker = await acquire(
|
||||
target({
|
||||
agentId: "worker",
|
||||
sessionId: "worker-session",
|
||||
storePath: path.join(root, "worker", "openclaw-agent.sqlite"),
|
||||
}),
|
||||
);
|
||||
await Promise.all([main.release(), worker.release()]);
|
||||
});
|
||||
|
||||
it("uses one key for session aliases and legacy/SQLite store locators", () => {
|
||||
const base = {
|
||||
agentId: "main",
|
||||
sessionId: "shared-session",
|
||||
sessionKey: "agent:main:alias-a",
|
||||
};
|
||||
expect(
|
||||
resolveSessionWriteLockTargetKey({ ...base, storePath: path.join(root, "sessions.json") }),
|
||||
).toBe(
|
||||
resolveSessionWriteLockTargetKey({
|
||||
...base,
|
||||
sessionKey: "agent:main:alias-b",
|
||||
storePath: path.join(root, "openclaw-agent.sqlite"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"keeps the key stable when a missing directory appears beneath a symlink",
|
||||
async () => {
|
||||
const realRoot = path.join(root, "real");
|
||||
const linkedRoot = path.join(root, "linked");
|
||||
await fs.mkdir(realRoot);
|
||||
await fs.symlink(realRoot, linkedRoot);
|
||||
const transcript = {
|
||||
agentId: "main",
|
||||
sessionId: "shared-session",
|
||||
sessionKey: "agent:main:main",
|
||||
storePath: path.join(linkedRoot, "new-agent", "sessions.json"),
|
||||
};
|
||||
const before = resolveSessionWriteLockTargetKey(transcript);
|
||||
await fs.mkdir(path.join(linkedRoot, "new-agent"));
|
||||
expect(resolveSessionWriteLockTargetKey(transcript)).toBe(before);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("session write lease options", () => {
|
||||
it("derives bounded max hold from timeout plus grace", () => {
|
||||
expect(resolveSessionLockMaxHoldFromTimeout({ timeoutMs: 600_000 })).toBe(720_000);
|
||||
expect(resolveSessionLockMaxHoldFromTimeout({ timeoutMs: 1_000, minMs: 5_000 })).toBe(121_000);
|
||||
expect(resolveSessionLockMaxHoldFromTimeout({ timeoutMs: Number.POSITIVE_INFINITY })).toBe(
|
||||
MAX_TIMER_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
|
||||
it("uses the built-in acquire timeout", () => {
|
||||
expect(resolveSessionWriteLockAcquireTimeoutMs()).toBe(60_000);
|
||||
});
|
||||
|
||||
it("accepts decimal emergency overrides", () => {
|
||||
expect(
|
||||
resolveSessionWriteLockOptions(undefined, {
|
||||
env: {
|
||||
OPENCLAW_SESSION_WRITE_LOCK_ACQUIRE_TIMEOUT_MS: "120000",
|
||||
OPENCLAW_SESSION_WRITE_LOCK_STALE_MS: "60000",
|
||||
OPENCLAW_SESSION_WRITE_LOCK_MAX_HOLD_MS: "50000",
|
||||
},
|
||||
}),
|
||||
).toEqual({ timeoutMs: 120_000, staleMs: 60_000, maxHoldMs: 50_000 });
|
||||
});
|
||||
|
||||
it("ignores non-decimal, unsafe, and unsupported infinite overrides", () => {
|
||||
expect(
|
||||
resolveSessionWriteLockOptions(undefined, {
|
||||
env: {
|
||||
OPENCLAW_SESSION_WRITE_LOCK_ACQUIRE_TIMEOUT_MS: "Infinity",
|
||||
OPENCLAW_SESSION_WRITE_LOCK_STALE_MS: "0x1000",
|
||||
OPENCLAW_SESSION_WRITE_LOCK_MAX_HOLD_MS: "9007199254740993",
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
timeoutMs: Number.POSITIVE_INFINITY,
|
||||
staleMs: 30 * 60_000,
|
||||
maxHoldMs: 5 * 60_000,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,403 +0,0 @@
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion";
|
||||
import type { SessionTranscriptRuntimeTarget } from "../config/sessions/session-accessor.types.js";
|
||||
import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { resolvePathViaExistingAncestorSync } from "../infra/boundary-path.js";
|
||||
import { readWindowsProcessStartTimeSync } from "../infra/windows-port-pids.js";
|
||||
import { LEGACY_IMPLICIT_AGENT_ID, resolveAgentIdFromSessionKey } from "../routing/session-key.js";
|
||||
import { resolveGlobalSingleton } from "../shared/global-singleton.js";
|
||||
import { getFileLockProcessStartTime, isPidAlive } from "../shared/pid-alive.js";
|
||||
import {
|
||||
openOpenClawAgentDatabase,
|
||||
type OpenClawAgentDatabaseOptions,
|
||||
} from "../state/openclaw-agent-db.js";
|
||||
import {
|
||||
acquireOpenClawStateLease,
|
||||
OpenClawStateLeaseError,
|
||||
type OpenClawStateLeaseHandle,
|
||||
} from "../state/openclaw-state-lease.js";
|
||||
import {
|
||||
SessionWriteLockStaleError,
|
||||
SessionWriteLockTimeoutError,
|
||||
} from "./session-write-lock-error.js";
|
||||
|
||||
const DEFAULT_SESSION_WRITE_LOCK_STALE_MS = 30 * 60 * 1000;
|
||||
const DEFAULT_SESSION_WRITE_LOCK_MAX_HOLD_MS = 5 * 60 * 1000;
|
||||
const DEFAULT_SESSION_WRITE_LOCK_ACQUIRE_TIMEOUT_MS = 60_000;
|
||||
const DEFAULT_TIMEOUT_GRACE_MS = 2 * 60 * 1000;
|
||||
const SESSION_WRITE_LEASE_SCOPE = "session-write";
|
||||
const SESSION_WRITE_LEASE_STATE_KEY = Symbol.for("openclaw.sessionWriteLeaseState");
|
||||
|
||||
const defaultProcessStartTimeForLock = (pid: number): number | null =>
|
||||
process.platform === "win32"
|
||||
? readWindowsProcessStartTimeSync(pid, 1_000)
|
||||
: getFileLockProcessStartTime(pid);
|
||||
let resolveProcessStartTimeForLock = defaultProcessStartTimeForLock;
|
||||
|
||||
type SessionWriteLeaseEntry = {
|
||||
lease: OpenClawStateLeaseHandle;
|
||||
refCount: number;
|
||||
};
|
||||
|
||||
const sessionWriteLeaseState = resolveGlobalSingleton(
|
||||
SESSION_WRITE_LEASE_STATE_KEY,
|
||||
(): { held: Map<string, SessionWriteLeaseEntry> } => ({ held: new Map() }),
|
||||
);
|
||||
|
||||
export type SessionWriteLockAcquireTimeoutConfig = OpenClawConfig;
|
||||
|
||||
type SessionWriteLockMsKey = "acquireTimeoutMs" | "staleMs" | "maxHoldMs";
|
||||
|
||||
const SESSION_WRITE_LOCK_ENV: Record<SessionWriteLockMsKey, string> = {
|
||||
acquireTimeoutMs: "OPENCLAW_SESSION_WRITE_LOCK_ACQUIRE_TIMEOUT_MS",
|
||||
staleMs: "OPENCLAW_SESSION_WRITE_LOCK_STALE_MS",
|
||||
maxHoldMs: "OPENCLAW_SESSION_WRITE_LOCK_MAX_HOLD_MS",
|
||||
};
|
||||
|
||||
function parsePositiveMs(
|
||||
value: number | undefined,
|
||||
options: { allowInfinity?: boolean } = {},
|
||||
): number | undefined {
|
||||
if (typeof value !== "number" || Number.isNaN(value) || value <= 0) {
|
||||
return undefined;
|
||||
}
|
||||
if (value === Number.POSITIVE_INFINITY) {
|
||||
return options.allowInfinity ? value : undefined;
|
||||
}
|
||||
return Number.isSafeInteger(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function resolvePositiveMs(
|
||||
value: number | undefined,
|
||||
fallback: number,
|
||||
options: { allowInfinity?: boolean } = {},
|
||||
): number {
|
||||
if (value === Number.POSITIVE_INFINITY) {
|
||||
return options.allowInfinity ? value : fallback;
|
||||
}
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
|
||||
}
|
||||
|
||||
function readPositiveMsEnv(
|
||||
env: NodeJS.ProcessEnv,
|
||||
key: string,
|
||||
options: { allowInfinity?: boolean } = {},
|
||||
): number | undefined {
|
||||
const raw = env[key]?.trim();
|
||||
if (!raw) {
|
||||
return undefined;
|
||||
}
|
||||
if (raw === "Infinity") {
|
||||
return options.allowInfinity ? Number.POSITIVE_INFINITY : undefined;
|
||||
}
|
||||
return /^\d+$/.test(raw) ? parsePositiveMs(Number(raw), options) : undefined;
|
||||
}
|
||||
|
||||
function resolveSessionWriteLockMs(params: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
key: SessionWriteLockMsKey;
|
||||
fallback: number;
|
||||
allowInfinity?: boolean;
|
||||
}): number {
|
||||
return (
|
||||
readPositiveMsEnv(params.env ?? process.env, SESSION_WRITE_LOCK_ENV[params.key], {
|
||||
allowInfinity: params.allowInfinity,
|
||||
}) ?? params.fallback
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveSessionWriteLockAcquireTimeoutMs(
|
||||
_config?: SessionWriteLockAcquireTimeoutConfig,
|
||||
env?: NodeJS.ProcessEnv,
|
||||
): number {
|
||||
return resolveSessionWriteLockMs({
|
||||
env,
|
||||
key: "acquireTimeoutMs",
|
||||
fallback: DEFAULT_SESSION_WRITE_LOCK_ACQUIRE_TIMEOUT_MS,
|
||||
allowInfinity: true,
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveSessionWriteLockStaleMs(
|
||||
_config?: SessionWriteLockAcquireTimeoutConfig,
|
||||
env?: NodeJS.ProcessEnv,
|
||||
): number {
|
||||
return resolveSessionWriteLockMs({
|
||||
env,
|
||||
key: "staleMs",
|
||||
fallback: DEFAULT_SESSION_WRITE_LOCK_STALE_MS,
|
||||
});
|
||||
}
|
||||
|
||||
function resolveSessionWriteLockMaxHoldMs(
|
||||
_config?: SessionWriteLockAcquireTimeoutConfig,
|
||||
params: { env?: NodeJS.ProcessEnv; fallback?: number } = {},
|
||||
): number {
|
||||
return resolveSessionWriteLockMs({
|
||||
env: params.env,
|
||||
key: "maxHoldMs",
|
||||
fallback: params.fallback ?? DEFAULT_SESSION_WRITE_LOCK_MAX_HOLD_MS,
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveSessionWriteLockOptions(
|
||||
config?: SessionWriteLockAcquireTimeoutConfig,
|
||||
params: { env?: NodeJS.ProcessEnv; maxHoldMsFallback?: number } = {},
|
||||
): { timeoutMs: number; staleMs: number; maxHoldMs: number } {
|
||||
return {
|
||||
timeoutMs: resolveSessionWriteLockAcquireTimeoutMs(config, params.env),
|
||||
staleMs: resolveSessionWriteLockStaleMs(config, params.env),
|
||||
maxHoldMs: resolveSessionWriteLockMaxHoldMs(config, {
|
||||
env: params.env,
|
||||
fallback: params.maxHoldMsFallback,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveSessionLockMaxHoldFromTimeout(params: {
|
||||
timeoutMs: number;
|
||||
graceMs?: number;
|
||||
minMs?: number;
|
||||
}): number {
|
||||
const minMs = resolvePositiveMs(params.minMs, DEFAULT_SESSION_WRITE_LOCK_MAX_HOLD_MS);
|
||||
const timeoutMs = resolvePositiveMs(params.timeoutMs, minMs, { allowInfinity: true });
|
||||
if (timeoutMs === Number.POSITIVE_INFINITY) {
|
||||
return MAX_TIMER_TIMEOUT_MS;
|
||||
}
|
||||
const graceMs = resolvePositiveMs(params.graceMs, DEFAULT_TIMEOUT_GRACE_MS);
|
||||
return Math.min(MAX_TIMER_TIMEOUT_MS, Math.max(minMs, timeoutMs + graceMs));
|
||||
}
|
||||
|
||||
export function resolveSessionWriteLockTargetKey(target: SessionTranscriptRuntimeTarget): string {
|
||||
const databaseTarget = resolveSqliteTargetFromSessionStorePath(target.storePath, {
|
||||
agentId: target.agentId,
|
||||
});
|
||||
return JSON.stringify([
|
||||
target.agentId,
|
||||
resolvePathViaExistingAncestorSync(databaseTarget.path),
|
||||
target.sessionId,
|
||||
]);
|
||||
}
|
||||
|
||||
function resolveSessionWriteLeaseDatabaseOptions(sessionKey: string): OpenClawAgentDatabaseOptions {
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(sessionKey) as unknown;
|
||||
} catch {
|
||||
// Shipped unqualified harness keys use their agent's canonical database.
|
||||
}
|
||||
if (
|
||||
Array.isArray(parsed) &&
|
||||
parsed.length === 3 &&
|
||||
parsed.every((value) => typeof value === "string" && value.trim().length > 0)
|
||||
) {
|
||||
const [agentId, storePath] = parsed as [string, string, string];
|
||||
const target = resolveSqliteTargetFromSessionStorePath(storePath, { agentId });
|
||||
return { agentId: target.agentId ?? agentId, path: target.path };
|
||||
}
|
||||
const agentId = resolveAgentIdFromSessionKey(sessionKey, LEGACY_IMPLICIT_AGENT_ID);
|
||||
return { agentId, path: openOpenClawAgentDatabase({ agentId }).path };
|
||||
}
|
||||
|
||||
function leaseDurationMs(value: number, minimum: number): number {
|
||||
if (!Number.isFinite(value)) {
|
||||
return MAX_TIMER_TIMEOUT_MS;
|
||||
}
|
||||
return Math.min(MAX_TIMER_TIMEOUT_MS, Math.max(minimum, Math.floor(value)));
|
||||
}
|
||||
|
||||
function leasePath(sessionKey: string): string {
|
||||
return `sqlite:${SESSION_WRITE_LEASE_SCOPE}:${sessionKey}`;
|
||||
}
|
||||
|
||||
function leaseLost(sessionKey: string): SessionWriteLockStaleError {
|
||||
return new SessionWriteLockStaleError({
|
||||
owner: "expired or replaced SQLite lease",
|
||||
lockPath: leasePath(sessionKey),
|
||||
staleReasons: ["lease-lost"],
|
||||
});
|
||||
}
|
||||
|
||||
function describeLeaseTimeoutOwner(error: OpenClawStateLeaseError): string {
|
||||
const payload = error.ownerPayload;
|
||||
if (!payload) {
|
||||
return "another OpenClaw process";
|
||||
}
|
||||
if (payload.pid === process.pid) {
|
||||
return `held by this process (pid ${payload.pid}); an earlier run has not released it`;
|
||||
}
|
||||
if (!isPidAlive(payload.pid)) {
|
||||
return "another OpenClaw process";
|
||||
}
|
||||
const observedStartTime = resolveProcessStartTimeForLock(payload.pid);
|
||||
if (
|
||||
payload.starttime !== undefined &&
|
||||
observedStartTime !== null &&
|
||||
payload.starttime !== observedStartTime
|
||||
) {
|
||||
return "another OpenClaw process";
|
||||
}
|
||||
return `held by OpenClaw process pid ${payload.pid}`;
|
||||
}
|
||||
|
||||
function createLeaseHandle(sessionKey: string, entry: SessionWriteLeaseEntry) {
|
||||
let released = false;
|
||||
let releasePromise: Promise<void> | undefined;
|
||||
return {
|
||||
assertOwned: () => {
|
||||
if (sessionWriteLeaseState.held.get(sessionKey) !== entry) {
|
||||
throw leaseLost(sessionKey);
|
||||
}
|
||||
try {
|
||||
entry.lease.assertOwned();
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof OpenClawStateLeaseError &&
|
||||
error.code === "OPENCLAW_STATE_LEASE_LOST"
|
||||
) {
|
||||
throw leaseLost(sessionKey);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
release: () => {
|
||||
if (released) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
releasePromise ??= (async () => {
|
||||
const current = sessionWriteLeaseState.held.get(sessionKey);
|
||||
if (current !== entry) {
|
||||
return;
|
||||
}
|
||||
if (current.refCount > 1) {
|
||||
current.refCount -= 1;
|
||||
return;
|
||||
}
|
||||
sessionWriteLeaseState.held.delete(sessionKey);
|
||||
try {
|
||||
await current.lease.release();
|
||||
} catch (error) {
|
||||
if (!sessionWriteLeaseState.held.has(sessionKey)) {
|
||||
sessionWriteLeaseState.held.set(sessionKey, current);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
})().then(
|
||||
() => {
|
||||
released = true;
|
||||
},
|
||||
(error: unknown) => {
|
||||
releasePromise = undefined;
|
||||
throw error;
|
||||
},
|
||||
);
|
||||
return releasePromise;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function acquireSessionWriteLock(params: {
|
||||
sessionFile: string;
|
||||
targetKind: "session-key";
|
||||
timeoutMs?: number;
|
||||
staleMs?: number;
|
||||
maxHoldMs?: number;
|
||||
signal?: AbortSignal;
|
||||
allowReentrant?: boolean;
|
||||
reentrantOwner?: never;
|
||||
}): Promise<{ assertOwned?: () => void; release: () => Promise<void> }> {
|
||||
if (params.signal?.aborted) {
|
||||
throw params.signal.reason;
|
||||
}
|
||||
const defaults = resolveSessionWriteLockOptions();
|
||||
const timeoutMs = resolvePositiveMs(params.timeoutMs, defaults.timeoutMs, {
|
||||
allowInfinity: true,
|
||||
});
|
||||
const maxHoldMs = resolvePositiveMs(params.maxHoldMs, defaults.maxHoldMs);
|
||||
const existing = sessionWriteLeaseState.held.get(params.sessionFile);
|
||||
if (params.allowReentrant === true && existing) {
|
||||
existing.refCount += 1;
|
||||
return createLeaseHandle(params.sessionFile, existing);
|
||||
}
|
||||
|
||||
const databaseOptions = resolveSessionWriteLeaseDatabaseOptions(params.sessionFile);
|
||||
openOpenClawAgentDatabase(databaseOptions);
|
||||
const admission = new AbortController();
|
||||
const abortAdmission = () => admission.abort(params.signal?.reason);
|
||||
params.signal?.addEventListener("abort", abortAdmission, { once: true });
|
||||
try {
|
||||
const lease = await acquireOpenClawStateLease({
|
||||
scope: SESSION_WRITE_LEASE_SCOPE,
|
||||
key: params.sessionFile,
|
||||
database: {
|
||||
scope: "agent",
|
||||
agentId: databaseOptions.agentId,
|
||||
...(databaseOptions.path ? { path: databaseOptions.path } : {}),
|
||||
},
|
||||
leaseMs: leaseDurationMs(maxHoldMs, 1_000),
|
||||
waitMs: leaseDurationMs(timeoutMs, 0),
|
||||
signal: admission.signal,
|
||||
leaseLabel: "session write lease",
|
||||
operationLabel: "session.write-lease",
|
||||
strictRelease: true,
|
||||
processOwner: {
|
||||
pid: process.pid,
|
||||
startTime: resolveProcessStartTimeForLock(process.pid),
|
||||
isAlive: isPidAlive,
|
||||
readStartTime: resolveProcessStartTimeForLock,
|
||||
},
|
||||
});
|
||||
params.signal?.removeEventListener("abort", abortAdmission);
|
||||
const entry = { lease, refCount: 1 };
|
||||
sessionWriteLeaseState.held.set(params.sessionFile, entry);
|
||||
return createLeaseHandle(params.sessionFile, entry);
|
||||
} catch (error) {
|
||||
if (params.signal?.aborted) {
|
||||
throw params.signal.reason;
|
||||
}
|
||||
if (error instanceof OpenClawStateLeaseError && error.code === "OPENCLAW_STATE_LEASE_TIMEOUT") {
|
||||
throw new SessionWriteLockTimeoutError({
|
||||
timeoutMs,
|
||||
owner: describeLeaseTimeoutOwner(error),
|
||||
lockPath: leasePath(params.sessionFile),
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
params.signal?.removeEventListener("abort", abortAdmission);
|
||||
}
|
||||
}
|
||||
|
||||
function releaseAllSessionWriteLeasesSynchronously(): void {
|
||||
for (const entry of sessionWriteLeaseState.held.values()) {
|
||||
try {
|
||||
entry.lease.releaseSynchronously();
|
||||
} catch {
|
||||
// A dead process owner remains reclaimable by pid/start-time identity.
|
||||
}
|
||||
}
|
||||
sessionWriteLeaseState.held.clear();
|
||||
}
|
||||
|
||||
export async function drainSessionWriteLockStateForTest(): Promise<void> {
|
||||
const held = [...sessionWriteLeaseState.held.values()];
|
||||
sessionWriteLeaseState.held.clear();
|
||||
await Promise.all(held.map((entry) => entry.lease.release().catch(() => undefined)));
|
||||
}
|
||||
|
||||
function resetSessionWriteLockStateForTest(): void {
|
||||
releaseAllSessionWriteLeasesSynchronously();
|
||||
resolveProcessStartTimeForLock = defaultProcessStartTimeForLock;
|
||||
}
|
||||
|
||||
if (process.env.VITEST || process.env.NODE_ENV === "test") {
|
||||
(globalThis as Record<PropertyKey, unknown>)[Symbol.for("openclaw.sessionWriteLockTestApi")] = {
|
||||
resetSessionWriteLockStateForTest,
|
||||
testing: {
|
||||
releaseAllLocksSync: releaseAllSessionWriteLeasesSynchronously,
|
||||
setProcessStartTimeResolverForTest(resolver: ((pid: number) => number | null) | null): void {
|
||||
resolveProcessStartTimeForLock = resolver ?? defaultProcessStartTimeForLock;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import type {
|
||||
AgentSessionConfig,
|
||||
AgentSessionEvent,
|
||||
AgentSessionEventListener,
|
||||
AgentSessionWriteLockRunner,
|
||||
AgentSessionWriteSettlementRunner,
|
||||
} from "./agent-session-types.js";
|
||||
import { extractTextContent } from "./agent-session-utils.js";
|
||||
import { formatNoApiKeyFoundMessage } from "./auth-guidance.js";
|
||||
@@ -109,7 +109,7 @@ export abstract class AgentSessionBase {
|
||||
protected disableBuiltInTools: boolean;
|
||||
protected baseToolsOverride?: Record<string, AgentTool>;
|
||||
protected sessionStartEvent: SessionStartEvent;
|
||||
protected withExternalSessionWriteLock?: AgentSessionWriteLockRunner;
|
||||
protected withExternalSessionWriteSettlement?: AgentSessionWriteSettlementRunner;
|
||||
protected extensionUIContext?: ExtensionUIContext;
|
||||
protected extensionCommandContextActions?: ExtensionCommandContextActions;
|
||||
protected extensionAbortHandler?: () => void;
|
||||
@@ -150,7 +150,7 @@ export abstract class AgentSessionBase {
|
||||
type: "session_start",
|
||||
reason: "startup",
|
||||
};
|
||||
this.withExternalSessionWriteLock = config.withSessionWriteLock;
|
||||
this.withExternalSessionWriteSettlement = config.withSessionWriteSettlement;
|
||||
this.contextOverflowRecoveryOwner = config.contextOverflowRecoveryOwner ?? "session";
|
||||
}
|
||||
|
||||
@@ -200,9 +200,9 @@ export abstract class AgentSessionBase {
|
||||
return result.ok ? { apiKey: result.apiKey, headers: result.headers } : {};
|
||||
}
|
||||
|
||||
protected async runWithSessionWriteLock<T>(run: () => Promise<T> | T): Promise<T> {
|
||||
return this.withExternalSessionWriteLock
|
||||
? await this.withExternalSessionWriteLock(run)
|
||||
protected async runWithSessionWriteSettlement<T>(run: () => Promise<T> | T): Promise<T> {
|
||||
return this.withExternalSessionWriteSettlement
|
||||
? await this.withExternalSessionWriteSettlement(run)
|
||||
: await run();
|
||||
}
|
||||
|
||||
@@ -221,7 +221,7 @@ export abstract class AgentSessionBase {
|
||||
protected installAgentToolHooks(): void {
|
||||
this.agent.beforeToolCall = async ({ toolCall, args }) => {
|
||||
const runner = this.currentExtensionRunner;
|
||||
return await this.runWithSessionWriteLock(async () => {
|
||||
return await this.runWithSessionWriteSettlement(async () => {
|
||||
if (!runner.hasHandlers("tool_call")) {
|
||||
return undefined;
|
||||
}
|
||||
@@ -248,7 +248,7 @@ export abstract class AgentSessionBase {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const hookResult = await this.runWithSessionWriteLock(
|
||||
const hookResult = await this.runWithSessionWriteSettlement(
|
||||
async () =>
|
||||
await runner.emitToolResult({
|
||||
type: "tool_result",
|
||||
@@ -323,7 +323,9 @@ export abstract class AgentSessionBase {
|
||||
(reason as { turnHandoff?: unknown }).turnHandoff === true;
|
||||
}
|
||||
if (this.eventMayWriteSession(event)) {
|
||||
await this.runWithSessionWriteLock(async () => await this.handleAgentEventUnlocked(event));
|
||||
await this.runWithSessionWriteSettlement(
|
||||
async () => await this.handleAgentEventUnlocked(event),
|
||||
);
|
||||
return;
|
||||
}
|
||||
await this.handleAgentEventUnlocked(event);
|
||||
|
||||
@@ -41,18 +41,19 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection {
|
||||
* @param customInstructions Optional instructions for the compaction summary
|
||||
*/
|
||||
async compact(customInstructions?: string): Promise<CompactionResult> {
|
||||
return await this.runWithSessionWriteLock(
|
||||
async () => await this.compactWithSessionWriteLock(customInstructions, "none"),
|
||||
return await this.runWithSessionWriteSettlement(
|
||||
async () => await this.compactWithSessionWriteSettlement(customInstructions, "none"),
|
||||
);
|
||||
}
|
||||
|
||||
async [agentSessionAutomaticCompaction](customInstructions?: string): Promise<CompactionResult> {
|
||||
return await this.runWithSessionWriteLock(
|
||||
async () => await this.compactWithSessionWriteLock(customInstructions, "retry-invalid-once"),
|
||||
return await this.runWithSessionWriteSettlement(
|
||||
async () =>
|
||||
await this.compactWithSessionWriteSettlement(customInstructions, "retry-invalid-once"),
|
||||
);
|
||||
}
|
||||
|
||||
private async compactWithSessionWriteLock(
|
||||
private async compactWithSessionWriteSettlement(
|
||||
customInstructions?: string,
|
||||
summaryOutputPolicy: SummaryOutputPolicy = "none",
|
||||
): Promise<CompactionResult> {
|
||||
|
||||
@@ -53,7 +53,7 @@ export type AgentSessionEvent =
|
||||
| { type: "auto_retry_end"; success: boolean; attempt: number; finalError?: string };
|
||||
|
||||
export type AgentSessionEventListener = (event: AgentSessionEvent) => unknown;
|
||||
export type AgentSessionWriteLockRunner = <T>(run: () => Promise<T> | T) => Promise<T>;
|
||||
export type AgentSessionWriteSettlementRunner = <T>(run: () => Promise<T> | T) => Promise<T>;
|
||||
|
||||
export interface AgentSessionConfig {
|
||||
agent: Agent;
|
||||
@@ -80,8 +80,8 @@ export interface AgentSessionConfig {
|
||||
extensionRunnerRef?: { current?: ExtensionRunner };
|
||||
/** Session start metadata emitted when extensions bind to this runtime. */
|
||||
sessionStartEvent?: SessionStartEvent;
|
||||
/** Lock used before session-file writes or write-capable hooks. */
|
||||
withSessionWriteLock?: AgentSessionWriteLockRunner;
|
||||
/** Settlement boundary for session writes and write-capable hooks. */
|
||||
withSessionWriteSettlement?: AgentSessionWriteSettlementRunner;
|
||||
/** Owner of reactive context-overflow recovery. Defaults to the session. */
|
||||
contextOverflowRecoveryOwner?: "session" | "caller";
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import path from "node:path";
|
||||
import { createAssistantMessageEventStream, type AssistantMessage } from "openclaw/plugin-sdk/llm";
|
||||
// Agent session SDK tests cover default tool wiring, prompt preservation, and
|
||||
// session write-lock behavior.
|
||||
// session write-settlement behavior.
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
|
||||
@@ -675,9 +675,9 @@ describe("createAgentSession tool defaults", () => {
|
||||
expect(exactPromptOptions.promptGuidelines).toEqual(["Use custom_lookup for test values."]);
|
||||
});
|
||||
|
||||
it("runs session message persistence under the configured write lock", async () => {
|
||||
// Transcript writes share the caller-provided lock so concurrent event
|
||||
// handlers cannot interleave JSONL persistence.
|
||||
it("runs session message persistence under the configured write settlement", async () => {
|
||||
// Transcript writes share the caller-provided settlement boundary so
|
||||
// concurrent event handlers cannot interleave persistence.
|
||||
const events: string[] = [];
|
||||
const sessionManager = SessionManager.inMemory();
|
||||
const { session } = await createAgentSession({
|
||||
@@ -686,12 +686,12 @@ describe("createAgentSession tool defaults", () => {
|
||||
sessionManager,
|
||||
settingsManager: SettingsManager.inMemory(),
|
||||
modelRegistry: ModelRegistry.inMemory(AuthStorage.inMemory()),
|
||||
withSessionWriteLock: async (run) => {
|
||||
events.push("lock:start");
|
||||
withSessionWriteSettlement: async (run) => {
|
||||
events.push("settlement:start");
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
events.push("lock:end");
|
||||
events.push("settlement:end");
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -709,11 +709,11 @@ describe("createAgentSession tool defaults", () => {
|
||||
},
|
||||
});
|
||||
|
||||
expect(events).toEqual(["lock:start", "lock:end"]);
|
||||
expect(events).toEqual(["settlement:start", "settlement:end"]);
|
||||
expect(sessionManager.getEntries().some((entry) => entry.type === "message")).toBe(true);
|
||||
});
|
||||
|
||||
it("runs write-capable tool hooks under the configured write lock", async () => {
|
||||
it("runs write-capable tool hooks under the configured write settlement", async () => {
|
||||
const events: string[] = [];
|
||||
const handlers = new Map<string, Array<(...args: unknown[]) => Promise<unknown>>>([
|
||||
[
|
||||
@@ -733,12 +733,12 @@ describe("createAgentSession tool defaults", () => {
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settingsManager: SettingsManager.inMemory(),
|
||||
modelRegistry: ModelRegistry.inMemory(AuthStorage.inMemory()),
|
||||
withSessionWriteLock: async (run) => {
|
||||
events.push("lock:start");
|
||||
withSessionWriteSettlement: async (run) => {
|
||||
events.push("settlement:start");
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
events.push("lock:end");
|
||||
events.push("settlement:end");
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -770,12 +770,12 @@ describe("createAgentSession tool defaults", () => {
|
||||
},
|
||||
});
|
||||
|
||||
expect(events).toEqual(["lock:start", "hook", "lock:end"]);
|
||||
expect(events).toEqual(["settlement:start", "hook", "settlement:end"]);
|
||||
});
|
||||
|
||||
it("fences tool execution when no extension hook is registered", async () => {
|
||||
// Write-capable tools still enter the lock even without hooks; the lock is
|
||||
// about shared session state, not just extension execution.
|
||||
// Write-capable tools still enter the settlement boundary even without hooks;
|
||||
// it covers shared session state, not just extension execution.
|
||||
const events: string[] = [];
|
||||
const { session } = await createAgentSession({
|
||||
model: testModel,
|
||||
@@ -783,12 +783,12 @@ describe("createAgentSession tool defaults", () => {
|
||||
sessionManager: SessionManager.inMemory(),
|
||||
settingsManager: SettingsManager.inMemory(),
|
||||
modelRegistry: ModelRegistry.inMemory(AuthStorage.inMemory()),
|
||||
withSessionWriteLock: async (run) => {
|
||||
events.push("lock:start");
|
||||
withSessionWriteSettlement: async (run) => {
|
||||
events.push("settlement:start");
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
events.push("lock:end");
|
||||
events.push("settlement:end");
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -820,7 +820,7 @@ describe("createAgentSession tool defaults", () => {
|
||||
},
|
||||
});
|
||||
|
||||
expect(events).toEqual(["lock:start", "lock:end"]);
|
||||
expect(events).toEqual(["settlement:start", "settlement:end"]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ import {
|
||||
type InternalBeforeToolBatchHook,
|
||||
} from "../runtime/internal-hooks.js";
|
||||
import type { AgentSessionConfig } from "./agent-session-types.js";
|
||||
import { AgentSession, type AgentSessionWriteLockRunner } from "./agent-session.js";
|
||||
import { AgentSession, type AgentSessionWriteSettlementRunner } from "./agent-session.js";
|
||||
import { formatNoModelsAvailableMessage } from "./auth-guidance.js";
|
||||
import { AuthStorage } from "./auth-storage.js";
|
||||
import { DEFAULT_THINKING_LEVEL } from "./defaults.js";
|
||||
@@ -123,8 +123,8 @@ export interface CreateAgentSessionOptions {
|
||||
settingsManager?: SettingsManager;
|
||||
/** Session start event metadata for extension runtime startup. */
|
||||
sessionStartEvent?: SessionStartEvent;
|
||||
/** Optional lock used before session-file writes or write-capable extension hooks. */
|
||||
withSessionWriteLock?: AgentSessionWriteLockRunner;
|
||||
/** Optional settlement boundary for session writes and write-capable extension hooks. */
|
||||
withSessionWriteSettlement?: AgentSessionWriteSettlementRunner;
|
||||
}
|
||||
|
||||
type CreateAgentSessionInternalOptions = Pick<
|
||||
@@ -463,8 +463,10 @@ async function createAgentSessionImpl(
|
||||
};
|
||||
|
||||
const extensionRunnerRef: { current?: ExtensionRunner } = {};
|
||||
const runWithSessionWriteLock = async <T>(run: () => Promise<T> | T): Promise<T> =>
|
||||
options.withSessionWriteLock ? await options.withSessionWriteLock(run) : await run();
|
||||
const runWithSessionWriteSettlement = async <T>(run: () => Promise<T> | T): Promise<T> =>
|
||||
options.withSessionWriteSettlement
|
||||
? await options.withSessionWriteSettlement(run)
|
||||
: await run();
|
||||
|
||||
const modelRegistryRuntime = getModelRegistryRuntime(modelRegistry);
|
||||
const agent: Agent = new Agent({
|
||||
@@ -500,7 +502,7 @@ async function createAgentSessionImpl(
|
||||
if (!runner?.hasHandlers("before_provider_request")) {
|
||||
return payload;
|
||||
}
|
||||
return await runWithSessionWriteLock(
|
||||
return await runWithSessionWriteSettlement(
|
||||
async () => await runner.emitBeforeProviderRequest(payload),
|
||||
);
|
||||
},
|
||||
@@ -510,7 +512,7 @@ async function createAgentSessionImpl(
|
||||
if (!runner?.hasHandlers("after_provider_response")) {
|
||||
return;
|
||||
}
|
||||
await runWithSessionWriteLock(
|
||||
await runWithSessionWriteSettlement(
|
||||
async () =>
|
||||
await runner.emit({
|
||||
type: "after_provider_response",
|
||||
@@ -568,7 +570,7 @@ async function createAgentSessionImpl(
|
||||
disableBuiltInTools,
|
||||
extensionRunnerRef,
|
||||
sessionStartEvent: options.sessionStartEvent,
|
||||
withSessionWriteLock: options.withSessionWriteLock,
|
||||
withSessionWriteSettlement: options.withSessionWriteSettlement,
|
||||
contextOverflowRecoveryOwner: internalOptions.contextOverflowRecoveryOwner,
|
||||
});
|
||||
const extensionsResult = resourceLoader.getExtensions();
|
||||
|
||||
@@ -26,8 +26,8 @@ type DeliveryDeps = {
|
||||
type Testing = {
|
||||
setDepsForTest(overrides?: Partial<DeliveryDeps>): void;
|
||||
hasAnnounceSendEvidence(error: unknown): boolean;
|
||||
hasSessionFileChangedAnnounceError(error: unknown): boolean;
|
||||
isSessionFileChangedAnnounceError(message: string): boolean;
|
||||
hasWriterClaimReboundAnnounceError(error: unknown): boolean;
|
||||
isWriterClaimReboundAnnounceError(error: unknown): boolean;
|
||||
};
|
||||
|
||||
function getTesting(): Testing {
|
||||
@@ -39,8 +39,8 @@ function getTesting(): Testing {
|
||||
export const testing: Testing = {
|
||||
setDepsForTest: (overrides) => getTesting().setDepsForTest(overrides),
|
||||
hasAnnounceSendEvidence: (error) => getTesting().hasAnnounceSendEvidence(error),
|
||||
hasSessionFileChangedAnnounceError: (error) =>
|
||||
getTesting().hasSessionFileChangedAnnounceError(error),
|
||||
isSessionFileChangedAnnounceError: (message) =>
|
||||
getTesting().isSessionFileChangedAnnounceError(message),
|
||||
hasWriterClaimReboundAnnounceError: (error) =>
|
||||
getTesting().hasWriterClaimReboundAnnounceError(error),
|
||||
isWriterClaimReboundAnnounceError: (error) =>
|
||||
getTesting().isWriterClaimReboundAnnounceError(error),
|
||||
};
|
||||
|
||||
@@ -2431,7 +2431,7 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
name: "does not deliver a failure notice after ambiguous persistence failure",
|
||||
createCallGateway: () =>
|
||||
vi.fn(async () => {
|
||||
throw new Error("SessionWriteLockTimeoutError: session file locked before agent run");
|
||||
throw new Error("gateway persistence failed before agent run");
|
||||
}) as unknown as typeof runtimeCallGateway,
|
||||
event: {
|
||||
childSessionKey: "music_generate:task-failed",
|
||||
@@ -2727,11 +2727,9 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps generated media queued for the session agent after a requester handoff lock", async () => {
|
||||
it("keeps generated media queued after requester handoff fails", async () => {
|
||||
const callGateway = vi.fn(async () => {
|
||||
throw new Error(
|
||||
"SessionWriteLockTimeoutError: session file locked (timeout 60000ms): pid=43",
|
||||
);
|
||||
throw new Error("requester handoff failed before dispatch");
|
||||
}) as unknown as typeof runtimeCallGateway;
|
||||
const queueEmbeddedAgentMessageWithOutcome = createQueueOutcomeSequenceMock([
|
||||
"transcript_commit_wait_unsupported",
|
||||
@@ -3790,15 +3788,18 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
expect(agentParams.sourceReplyDeliveryMode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not retry session-file-changed failures with send evidence", async () => {
|
||||
it("does not retry writer-claim rebound failures with send evidence", async () => {
|
||||
const sendErr = new OutboundDeliveryError("outbound delivery failed", {
|
||||
cause: new Error("outbound delivery failed"),
|
||||
results: [{ channel: "telegram", messageId: "msg-1" }],
|
||||
});
|
||||
const callGateway: typeof runtimeCallGateway = vi.fn(async () => {
|
||||
throw new Error("session file changed while embedded prompt lock was released", {
|
||||
cause: sendErr,
|
||||
});
|
||||
throw Object.assign(
|
||||
new Error("session writer claim changed before transcript persistence", {
|
||||
cause: sendErr,
|
||||
}),
|
||||
{ name: "SessionTranscriptWriterClaimReboundError" },
|
||||
);
|
||||
});
|
||||
const queueEmbeddedAgentMessageWithOutcome = createQueueOutcomeSequenceMock(["no_active_run"]);
|
||||
const result = await deliverSlackChannelAnnouncement({
|
||||
@@ -3817,41 +3818,7 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
expect(queueEmbeddedAgentMessageWithOutcome).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not fallback-steer after wrapped prompt-lock takeover with send evidence", async () => {
|
||||
const takeoverErr = Object.assign(
|
||||
new Error("session file changed while embedded prompt lock was released: /tmp/session.jsonl"),
|
||||
{ name: "EmbeddedAttemptSessionTakeoverError" },
|
||||
);
|
||||
|
||||
const promptErr = Object.assign(new Error("some model error"), { visibleReplySent: true });
|
||||
const wrapperErr = Object.assign(new Error("some model error", { cause: takeoverErr }), {
|
||||
name: "EmbeddedAttemptSessionTakeoverError",
|
||||
cleanupError: takeoverErr,
|
||||
promptError: promptErr,
|
||||
});
|
||||
|
||||
const callGateway: typeof runtimeCallGateway = vi.fn(async () => {
|
||||
throw wrapperErr;
|
||||
});
|
||||
const queueEmbeddedAgentMessageWithOutcome = createQueueOutcomeSequenceMock(["no_active_run"]);
|
||||
const result = await deliverSlackChannelAnnouncement({
|
||||
callGateway,
|
||||
queueEmbeddedAgentMessageWithOutcome,
|
||||
sessionId: "requester-session-lock-race-wrapped-evidence",
|
||||
isActive: true,
|
||||
directIdempotencyKey: "announce-permanent-wrapped-lock-error-evidence",
|
||||
});
|
||||
|
||||
expect(result.delivered).toBe(false);
|
||||
expect(result.path).toBe("direct");
|
||||
expect(result.error).toBe("some model error");
|
||||
expect(result.disposition).toBe("ambiguous");
|
||||
expect(result.phases?.map((phase) => phase.phase)).toEqual(["direct-primary"]);
|
||||
expect(callGateway).toHaveBeenCalledTimes(1);
|
||||
expect(queueEmbeddedAgentMessageWithOutcome).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("retries session-file-changed failures without send evidence", async () => {
|
||||
it("retries writer-claim rebound failures without send evidence", async () => {
|
||||
let attempts = 0;
|
||||
const callGatewaySpy = vi.fn();
|
||||
const callGateway: typeof runtimeCallGateway = async <
|
||||
@@ -3860,7 +3827,10 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
callGatewaySpy();
|
||||
attempts++;
|
||||
if (attempts <= 1) {
|
||||
throw new Error("session file changed while embedded prompt lock was released");
|
||||
throw Object.assign(
|
||||
new Error("session writer claim changed before transcript persistence"),
|
||||
{ name: "SessionTranscriptWriterClaimReboundError" },
|
||||
);
|
||||
}
|
||||
return {
|
||||
result: {
|
||||
@@ -3932,63 +3902,47 @@ describe("deliverSubagentAnnouncement completion delivery", () => {
|
||||
expect(sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("detects send evidence from OutboundDeliveryError in the error chain", () => {
|
||||
const err = new Error(
|
||||
"session file changed while embedded prompt lock was released: /tmp/session.jsonl",
|
||||
{
|
||||
it("detects send evidence from OutboundDeliveryError in a writer rebound chain", () => {
|
||||
const err = Object.assign(
|
||||
new Error("session writer claim changed before transcript persistence", {
|
||||
cause: new OutboundDeliveryError("outbound delivery failed", {
|
||||
cause: new Error("outbound delivery failed"),
|
||||
results: [{ channel: "telegram", messageId: "msg-1" }],
|
||||
}),
|
||||
},
|
||||
}),
|
||||
{ name: "SessionTranscriptWriterClaimReboundError" },
|
||||
);
|
||||
|
||||
expect(testing.isSessionFileChangedAnnounceError(err.message)).toBe(true);
|
||||
expect(testing.isWriterClaimReboundAnnounceError(err)).toBe(true);
|
||||
expect(testing.hasAnnounceSendEvidence(err)).toBe(true);
|
||||
});
|
||||
|
||||
it("classifies session-file-changed error as no-send-evidence when the error chain has no send markers", () => {
|
||||
const err = new Error(
|
||||
"session file changed while embedded prompt lock was released: /tmp/session.jsonl",
|
||||
it("classifies writer rebound without send markers as no-send-evidence", () => {
|
||||
const err = Object.assign(
|
||||
new Error("session writer claim changed before transcript persistence"),
|
||||
{ name: "SessionTranscriptWriterClaimReboundError" },
|
||||
);
|
||||
|
||||
expect(testing.isSessionFileChangedAnnounceError(err.message)).toBe(true);
|
||||
expect(testing.isWriterClaimReboundAnnounceError(err)).toBe(true);
|
||||
expect(testing.hasAnnounceSendEvidence(err)).toBe(false);
|
||||
});
|
||||
|
||||
it("detects send evidence from visibleReplySent flag on session-file-changed error", () => {
|
||||
it("detects send evidence from visibleReplySent on writer rebound", () => {
|
||||
const err = Object.assign(
|
||||
new Error("session file changed while embedded prompt lock was released: /tmp/session.jsonl"),
|
||||
{ visibleReplySent: true },
|
||||
new Error("session writer claim changed before transcript persistence"),
|
||||
{ name: "SessionTranscriptWriterClaimReboundError", visibleReplySent: true },
|
||||
);
|
||||
|
||||
expect(testing.hasAnnounceSendEvidence(err)).toBe(true);
|
||||
});
|
||||
|
||||
it("detects send evidence from sentBeforeError flag on session-file-changed error", () => {
|
||||
it("detects send evidence from sentBeforeError on writer rebound", () => {
|
||||
const err = Object.assign(
|
||||
new Error("session file changed while embedded prompt lock was released: /tmp/session.jsonl"),
|
||||
{ sentBeforeError: true },
|
||||
new Error("session writer claim changed before transcript persistence"),
|
||||
{ name: "SessionTranscriptWriterClaimReboundError", sentBeforeError: true },
|
||||
);
|
||||
|
||||
expect(testing.hasAnnounceSendEvidence(err)).toBe(true);
|
||||
});
|
||||
|
||||
it("detects send evidence recursively through promptError", () => {
|
||||
const takeoverErr = Object.assign(
|
||||
new Error("session file changed while embedded prompt lock was released: /tmp/session.jsonl"),
|
||||
{ name: "EmbeddedAttemptSessionTakeoverError" },
|
||||
);
|
||||
|
||||
const promptErr = Object.assign(new Error("some model error"), { visibleReplySent: true });
|
||||
|
||||
const wrapperErr = Object.assign(new Error("some model error", { cause: takeoverErr }), {
|
||||
name: "EmbeddedAttemptSessionTakeoverError",
|
||||
promptError: promptErr,
|
||||
});
|
||||
|
||||
expect(testing.hasAnnounceSendEvidence(wrapperErr)).toBe(true);
|
||||
expect(testing.hasSessionFileChangedAnnounceError(wrapperErr)).toBe(true);
|
||||
});
|
||||
});
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
|
||||
@@ -373,8 +373,8 @@ const TRANSIENT_ANNOUNCE_DELIVERY_ERROR_PATTERNS: readonly RegExp[] = [
|
||||
/\b(econnreset|econnrefused|etimedout|enotfound|ehostunreach|network error)\b/i,
|
||||
];
|
||||
|
||||
const SESSION_FILE_CHANGED_ANNOUNCE_RE =
|
||||
/session file changed while embedded prompt lock was released/i;
|
||||
const WRITER_CLAIM_REBOUND_ANNOUNCE_RE =
|
||||
/session writer claim changed before transcript persistence/i;
|
||||
|
||||
const PERMANENT_ANNOUNCE_DELIVERY_ERROR_PATTERNS: readonly RegExp[] = [
|
||||
/unsupported channel/i,
|
||||
@@ -386,20 +386,19 @@ const PERMANENT_ANNOUNCE_DELIVERY_ERROR_PATTERNS: readonly RegExp[] = [
|
||||
/forbidden: bot was kicked/i,
|
||||
/recipient is not a valid/i,
|
||||
/outbound not configured for channel/i,
|
||||
SESSION_FILE_CHANGED_ANNOUNCE_RE,
|
||||
WRITER_CLAIM_REBOUND_ANNOUNCE_RE,
|
||||
];
|
||||
|
||||
function isSessionFileChangedAnnounceError(message: string): boolean {
|
||||
return SESSION_FILE_CHANGED_ANNOUNCE_RE.test(message);
|
||||
function isWriterClaimReboundAnnounceError(error: unknown): boolean {
|
||||
return Boolean(
|
||||
(error &&
|
||||
typeof error === "object" &&
|
||||
(error as { name?: unknown }).name === "SessionTranscriptWriterClaimReboundError") ||
|
||||
WRITER_CLAIM_REBOUND_ANNOUNCE_RE.test(summarizeDeliveryError(error)),
|
||||
);
|
||||
}
|
||||
|
||||
const ANNOUNCE_ERROR_CHAIN_KEYS = [
|
||||
"cause",
|
||||
"cleanupError",
|
||||
"error",
|
||||
"promptError",
|
||||
"reason",
|
||||
] as const;
|
||||
const ANNOUNCE_ERROR_CHAIN_KEYS = ["cause", "error", "reason"] as const;
|
||||
type AnnounceErrorChainKey = (typeof ANNOUNCE_ERROR_CHAIN_KEYS)[number];
|
||||
type AnnounceErrorRecord = Partial<Record<AnnounceErrorChainKey, unknown>> & {
|
||||
sentBeforeError?: unknown;
|
||||
@@ -429,10 +428,8 @@ function hasAnnounceErrorMatch(
|
||||
return ANNOUNCE_ERROR_CHAIN_KEYS.some((key) => hasAnnounceErrorMatch(error[key], matches, seen));
|
||||
}
|
||||
|
||||
function hasSessionFileChangedAnnounceError(error: unknown): boolean {
|
||||
return hasAnnounceErrorMatch(error, (candidate) =>
|
||||
isSessionFileChangedAnnounceError(summarizeDeliveryError(candidate)),
|
||||
);
|
||||
function hasWriterClaimReboundAnnounceError(error: unknown): boolean {
|
||||
return hasAnnounceErrorMatch(error, isWriterClaimReboundAnnounceError);
|
||||
}
|
||||
|
||||
function isTransientAnnounceDeliveryError(error: unknown): boolean {
|
||||
@@ -440,12 +437,12 @@ function isTransientAnnounceDeliveryError(error: unknown): boolean {
|
||||
const topLevelPermanent = Boolean(
|
||||
message && PERMANENT_ANNOUNCE_DELIVERY_ERROR_PATTERNS.some((re) => re.test(message)),
|
||||
);
|
||||
if (topLevelPermanent && !isSessionFileChangedAnnounceError(message)) {
|
||||
if (topLevelPermanent && !isWriterClaimReboundAnnounceError(error)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const sessionFileChanged = hasSessionFileChangedAnnounceError(error);
|
||||
if (sessionFileChanged) {
|
||||
const writerClaimRebound = hasWriterClaimReboundAnnounceError(error);
|
||||
if (writerClaimRebound) {
|
||||
return !hasAnnounceSendEvidence(error);
|
||||
}
|
||||
|
||||
@@ -474,7 +471,7 @@ function isPermanentAnnounceDeliveryError(error: unknown): boolean {
|
||||
const message = summarizeDeliveryError(error);
|
||||
return (
|
||||
(message && PERMANENT_ANNOUNCE_DELIVERY_ERROR_PATTERNS.some((re) => re.test(message))) ||
|
||||
hasSessionFileChangedAnnounceError(error)
|
||||
hasWriterClaimReboundAnnounceError(error)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1568,8 +1565,8 @@ const testing = {
|
||||
: defaultSubagentAnnounceDeliveryDeps;
|
||||
},
|
||||
hasAnnounceSendEvidence,
|
||||
hasSessionFileChangedAnnounceError,
|
||||
isSessionFileChangedAnnounceError,
|
||||
hasWriterClaimReboundAnnounceError,
|
||||
isWriterClaimReboundAnnounceError,
|
||||
};
|
||||
if (process.env.VITEST || process.env.NODE_ENV === "test") {
|
||||
(globalThis as Record<PropertyKey, unknown>)[
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { resolveSessionStorePathForScope } from "../config/sessions/session-store-path.js";
|
||||
import {
|
||||
runWithOwnedSessionTranscriptWriteLock,
|
||||
runWithOwnedSessionTranscriptWrite,
|
||||
withOwnedSessionTranscriptWrites,
|
||||
} from "../config/sessions/transcript-write-context.js";
|
||||
import type { CallGatewayOptions } from "../gateway/call.js";
|
||||
@@ -564,9 +564,9 @@ describe("subagent registry lifecycle hardening", () => {
|
||||
const cleanupReady = new Promise<void>((resolve) => {
|
||||
releaseCleanup = resolve;
|
||||
});
|
||||
const staleWriteLock = vi.fn();
|
||||
const withStaleWriteLock = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
staleWriteLock();
|
||||
const requesterTranscriptWrite = vi.fn();
|
||||
const withRequesterTranscriptWrite = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
requesterTranscriptWrite();
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before transcript write");
|
||||
}
|
||||
@@ -575,13 +575,13 @@ describe("subagent registry lifecycle hardening", () => {
|
||||
const freshTranscriptWrite = vi.fn(async () => {});
|
||||
const runSubagentAnnounceFlow = vi.fn(async () => {
|
||||
await cleanupReady;
|
||||
await runWithOwnedSessionTranscriptWriteLock({ sessionKey }, freshTranscriptWrite);
|
||||
await runWithOwnedSessionTranscriptWrite({ sessionKey }, freshTranscriptWrite);
|
||||
return true;
|
||||
});
|
||||
const controller = createLifecycleController({ entry, runSubagentAnnounceFlow });
|
||||
|
||||
await withOwnedSessionTranscriptWrites(
|
||||
{ sessionKey, assertOwned: () => undefined, withSessionWriteLock: withStaleWriteLock },
|
||||
{ sessionKey, withTranscriptWrite: withRequesterTranscriptWrite },
|
||||
async () => {
|
||||
expect(controller.startSubagentAnnounceCleanupFlow(entry.runId, entry)).toBe(true);
|
||||
},
|
||||
@@ -592,7 +592,7 @@ describe("subagent registry lifecycle hardening", () => {
|
||||
|
||||
await waitForLifecycleState(() => expect(freshTranscriptWrite).toHaveBeenCalledOnce());
|
||||
await waitForLifecycleState(() => expect(entry.delivery?.status).toBe("delivered"));
|
||||
expect(staleWriteLock).not.toHaveBeenCalled();
|
||||
expect(requesterTranscriptWrite).not.toHaveBeenCalled();
|
||||
expect(runSubagentAnnounceFlow).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
@@ -3836,9 +3836,9 @@ describe("requester settle wake trigger", () => {
|
||||
const wakeReady = new Promise<void>((resolve) => {
|
||||
releaseWake = resolve;
|
||||
});
|
||||
const staleWriteLock = vi.fn();
|
||||
const withStaleWriteLock = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
staleWriteLock();
|
||||
const requesterTranscriptWrite = vi.fn();
|
||||
const withRequesterTranscriptWrite = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
requesterTranscriptWrite();
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before transcript write");
|
||||
}
|
||||
@@ -3847,7 +3847,7 @@ describe("requester settle wake trigger", () => {
|
||||
const freshTranscriptWrite = vi.fn(async () => {});
|
||||
const settleWake = vi.fn(async () => {
|
||||
await wakeReady;
|
||||
await runWithOwnedSessionTranscriptWriteLock({ sessionKey }, freshTranscriptWrite);
|
||||
await runWithOwnedSessionTranscriptWrite({ sessionKey }, freshTranscriptWrite);
|
||||
return false;
|
||||
});
|
||||
const controller = createLifecycleController({
|
||||
@@ -3856,7 +3856,7 @@ describe("requester settle wake trigger", () => {
|
||||
});
|
||||
|
||||
await withOwnedSessionTranscriptWrites(
|
||||
{ sessionKey, assertOwned: () => undefined, withSessionWriteLock: withStaleWriteLock },
|
||||
{ sessionKey, withTranscriptWrite: withRequesterTranscriptWrite },
|
||||
async () => {
|
||||
controller.completeCleanupBookkeeping({
|
||||
runId: entry.runId,
|
||||
@@ -3871,7 +3871,7 @@ describe("requester settle wake trigger", () => {
|
||||
releaseWake();
|
||||
|
||||
await waitForLifecycleState(() => expect(freshTranscriptWrite).toHaveBeenCalledOnce());
|
||||
expect(staleWriteLock).not.toHaveBeenCalled();
|
||||
expect(requesterTranscriptWrite).not.toHaveBeenCalled();
|
||||
expect(settleWake).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ import type {
|
||||
SessionEntryPatchOptions,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import {
|
||||
runWithOwnedSessionTranscriptWriteLock,
|
||||
runWithOwnedSessionTranscriptWrite,
|
||||
withOwnedSessionTranscriptWrites,
|
||||
} from "../config/sessions/transcript-write-context.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
@@ -1992,9 +1992,9 @@ describe("subagent registry seam flow", () => {
|
||||
const pendingWait = new Promise<Record<string, unknown>>((resolve) => {
|
||||
resolveWait = resolve;
|
||||
});
|
||||
const staleWriteLock = vi.fn();
|
||||
const withStaleWriteLock = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
staleWriteLock();
|
||||
const requesterTranscriptWrite = vi.fn();
|
||||
const withRequesterTranscriptWrite = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
requesterTranscriptWrite();
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before transcript write");
|
||||
}
|
||||
@@ -2008,16 +2008,16 @@ describe("subagent registry seam flow", () => {
|
||||
return {};
|
||||
}
|
||||
const result = await pendingWait;
|
||||
await runWithOwnedSessionTranscriptWriteLock({ sessionKey }, freshCompletionWrite);
|
||||
await runWithOwnedSessionTranscriptWrite({ sessionKey }, freshCompletionWrite);
|
||||
return result;
|
||||
});
|
||||
mocks.runSubagentAnnounceFlow.mockImplementation(async () => {
|
||||
await runWithOwnedSessionTranscriptWriteLock({ sessionKey }, freshTranscriptWrite);
|
||||
await runWithOwnedSessionTranscriptWrite({ sessionKey }, freshTranscriptWrite);
|
||||
return true;
|
||||
});
|
||||
|
||||
await withOwnedSessionTranscriptWrites(
|
||||
{ sessionKey, assertOwned: () => undefined, withSessionWriteLock: withStaleWriteLock },
|
||||
{ sessionKey, withTranscriptWrite: withRequesterTranscriptWrite },
|
||||
async () => {
|
||||
mod.registerSubagentRun({
|
||||
runId: "run-detached-requester-owner",
|
||||
@@ -2039,7 +2039,7 @@ describe("subagent registry seam flow", () => {
|
||||
await waitForFast(() => expect(freshTranscriptWrite).toHaveBeenCalledOnce());
|
||||
expect(freshCompletionWrite).toHaveBeenCalledOnce();
|
||||
expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledOnce();
|
||||
expect(staleWriteLock).not.toHaveBeenCalled();
|
||||
expect(requesterTranscriptWrite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not fall back to network recovery without an instance-bound runtime", async () => {
|
||||
@@ -4755,7 +4755,7 @@ describe("subagent registry seam flow", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("does not restore session-write ownership after a successor is released", async () => {
|
||||
it("does not restore a superseded lifecycle after a successor is released", async () => {
|
||||
const childSessionKey = "agent:main:subagent:released-timing-owner";
|
||||
mockPendingAgentWait();
|
||||
mocks.loadSessionStore.mockReturnValue({
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// wake delivery, and direct media fallback behavior.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
runWithOwnedSessionTranscriptWriteLock,
|
||||
runWithOwnedSessionTranscriptWrite,
|
||||
withOwnedSessionTranscriptWrites,
|
||||
} from "../../config/sessions/transcript-write-context.js";
|
||||
import type { SessionEntry } from "../../config/sessions/types.js";
|
||||
@@ -129,9 +129,9 @@ describe("scheduleMediaGenerationTaskCompletion", () => {
|
||||
releaseBackground = resolve;
|
||||
});
|
||||
let scheduled: Promise<void> | undefined;
|
||||
const staleWriteLock = vi.fn();
|
||||
const withStaleWriteLock = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
staleWriteLock();
|
||||
const requesterTranscriptWrite = vi.fn();
|
||||
const withRequesterTranscriptWrite = async <T>(operation: () => Promise<T> | T): Promise<T> => {
|
||||
requesterTranscriptWrite();
|
||||
if (disposed) {
|
||||
throw new Error("attempt disposed before transcript write");
|
||||
}
|
||||
@@ -139,7 +139,7 @@ describe("scheduleMediaGenerationTaskCompletion", () => {
|
||||
};
|
||||
const freshTranscriptWrite = vi.fn(async () => {});
|
||||
const wakeTaskCompletion = vi.fn(async () => {
|
||||
await runWithOwnedSessionTranscriptWriteLock({ sessionKey }, freshTranscriptWrite);
|
||||
await runWithOwnedSessionTranscriptWrite({ sessionKey }, freshTranscriptWrite);
|
||||
return { status: "delivered" as const };
|
||||
});
|
||||
const completeTaskRun = vi.fn();
|
||||
@@ -159,7 +159,7 @@ describe("scheduleMediaGenerationTaskCompletion", () => {
|
||||
}));
|
||||
|
||||
await withOwnedSessionTranscriptWrites(
|
||||
{ sessionKey, assertOwned: () => undefined, withSessionWriteLock: withStaleWriteLock },
|
||||
{ sessionKey, withTranscriptWrite: withRequesterTranscriptWrite },
|
||||
async () => {
|
||||
scheduleMediaGenerationTaskCompletion({
|
||||
lifecycle,
|
||||
@@ -188,7 +188,7 @@ describe("scheduleMediaGenerationTaskCompletion", () => {
|
||||
}
|
||||
await scheduled;
|
||||
|
||||
expect(staleWriteLock).not.toHaveBeenCalled();
|
||||
expect(requesterTranscriptWrite).not.toHaveBeenCalled();
|
||||
expect(run).toHaveBeenCalledOnce();
|
||||
expect(freshTranscriptWrite).toHaveBeenCalledOnce();
|
||||
expect(wakeTaskCompletion).toHaveBeenCalledOnce();
|
||||
|
||||
@@ -405,8 +405,7 @@ describe("runSessionsSendA2AFlow announce delivery", () => {
|
||||
it("notifies the requester when delayed target delivery fails after acceptance", async () => {
|
||||
vi.mocked(waitForAgentRun).mockResolvedValueOnce({
|
||||
status: "timeout",
|
||||
error:
|
||||
"SessionWriteLockTimeoutError: session file locked (timeout 60000ms): pid=43 alive=true",
|
||||
error: "target run failed after delivery acceptance",
|
||||
pendingError: true,
|
||||
});
|
||||
|
||||
@@ -435,15 +434,14 @@ describe("runSessionsSendA2AFlow announce delivery", () => {
|
||||
});
|
||||
const stepInput = firstMockArg(vi.mocked(runAgentStep), "agent step");
|
||||
expect(stepInput.message).toContain("sessions_send delivery to");
|
||||
expect(stepInput.message).toContain("SessionWriteLockTimeoutError");
|
||||
expect(stepInput.message).toContain("target run failed after delivery acceptance");
|
||||
expect(gatewayCalls.find((call) => call.method === "send")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not notify the requester for waited sends that already returned the error inline", async () => {
|
||||
vi.mocked(waitForAgentRun).mockResolvedValueOnce({
|
||||
status: "timeout",
|
||||
error:
|
||||
"SessionWriteLockTimeoutError: session file locked (timeout 60000ms): pid=43 alive=true",
|
||||
error: "target run failed after delivery acceptance",
|
||||
pendingError: true,
|
||||
});
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@ import {
|
||||
AGENT_RUN_RESTART_ABORT_STOP_REASON,
|
||||
resolveAgentRunErrorLifecycleFields,
|
||||
} from "../../agents/run-termination.js";
|
||||
import { isSessionWriteLockLeaseLostError } from "../../agents/session-write-lock-error.js";
|
||||
import { logVerbose } from "../../globals.js";
|
||||
import { emitAgentEvent } from "../../infra/agent-events.js";
|
||||
import { sleepWithAbort } from "../../infra/backoff.js";
|
||||
@@ -109,24 +108,6 @@ type ErrorAction =
|
||||
| { kind: "retry"; liveModelSwitchError?: LiveSessionModelSwitchError }
|
||||
| Extract<AgentTurnInternalResult, { kind: "final" }>;
|
||||
|
||||
function isSessionLeaseLoss(error: unknown): boolean {
|
||||
const pending = [error];
|
||||
const seen = new Set<unknown>();
|
||||
for (const candidate of pending) {
|
||||
if (!candidate || seen.has(candidate)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(candidate);
|
||||
if (isSessionWriteLockLeaseLostError(candidate)) {
|
||||
return true;
|
||||
}
|
||||
if (candidate instanceof Error && "cause" in candidate && candidate.cause !== undefined) {
|
||||
pending.push(candidate.cause);
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function handleAgentExecutionError(params: {
|
||||
turn: AgentTurnParams;
|
||||
error: unknown;
|
||||
@@ -253,16 +234,6 @@ export async function handleAgentExecutionError(params: {
|
||||
if (replyOperationAbortAction) {
|
||||
return replyOperationAbortAction;
|
||||
}
|
||||
if (
|
||||
isSessionLeaseLoss(err) &&
|
||||
(await turn.confirmRestartRecoveryArmedAfterLeaseLoss?.()) === true
|
||||
) {
|
||||
// The replacement owns recovery only after the latest SQLite row confirms
|
||||
// the active claim or its terminal marker. The old owner then exits silently.
|
||||
turn.replyOperation?.abortForRestart();
|
||||
takePendingLifecycleTerminal()?.emit("end", err);
|
||||
return { kind: "final", payload: { text: SILENT_REPLY_TOKEN } };
|
||||
}
|
||||
const restartLifecycleError = resolveRestartLifecycleError(err);
|
||||
if (
|
||||
restartLifecycleError instanceof GatewayDrainingError ||
|
||||
|
||||
@@ -73,9 +73,6 @@ type ExecutePreparedReplyAgentRunInput = Pick<
|
||||
checkpointBeforeAgentReply: ReturnType<
|
||||
typeof createReplyRestartRecoveryClaimController
|
||||
>["checkpointBeforeAgentReply"];
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss: ReturnType<
|
||||
typeof createReplyRestartRecoveryClaimController
|
||||
>["confirmRestartRecoveryArmedAfterLeaseLoss"];
|
||||
getActiveIsNewSession: () => boolean;
|
||||
getActiveSessionEntry: () => SessionEntry | undefined;
|
||||
isHeartbeat: boolean;
|
||||
@@ -114,7 +111,6 @@ export async function executePreparedReplyAgentRun(
|
||||
blockStreamingEnabled,
|
||||
cfg,
|
||||
checkpointBeforeAgentReply: checkpointBeforeAgentReplyWithRecovery,
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss,
|
||||
commandBody,
|
||||
defaultModel,
|
||||
followupRun,
|
||||
@@ -381,7 +377,6 @@ export async function executePreparedReplyAgentRun(
|
||||
resolvedVerboseLevel,
|
||||
toolProgressDetail,
|
||||
replyMediaContext,
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss,
|
||||
isRestartRecoveryArmed,
|
||||
}),
|
||||
),
|
||||
@@ -479,7 +474,6 @@ export function createReplyAgentRestartRecoveryController(
|
||||
beginBeforeAgentReply,
|
||||
checkpointBeforeAgentReply,
|
||||
clear: clearRestartRecoveryDeliveryClaim,
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss,
|
||||
isArmed: isRestartRecoveryArmed,
|
||||
} = createReplyRestartRecoveryClaimController({
|
||||
admissionRunId:
|
||||
@@ -548,7 +542,6 @@ export function createReplyAgentRestartRecoveryController(
|
||||
admitUserTurn,
|
||||
beginBeforeAgentReply,
|
||||
checkpointBeforeAgentReply,
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss,
|
||||
clear: clearRestartRecoveryDeliveryClaim,
|
||||
isArmed: isRestartRecoveryArmed,
|
||||
};
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { FailoverError } from "../../agents/failover-error.js";
|
||||
import { AgentHarnessSessionSupersededError } from "../../agents/harness/errors.js";
|
||||
import { SessionWriteLockStaleError } from "../../agents/session-write-lock-error.js";
|
||||
import { CommandLaneClearedError, GatewayDrainingError } from "../../process/command-queue.js";
|
||||
import { getReplyPayloadMetadata } from "../reply-payload.js";
|
||||
import type { TemplateContext } from "../templating.js";
|
||||
@@ -382,55 +381,6 @@ describe("executeAgentTurn: terminal failures", () => {
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("hands a confirmed restart lease loss to the replacement owner", async () => {
|
||||
const agentEvents = await import("../../infra/agent-events.js");
|
||||
const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent);
|
||||
const { replyOperation, failMock } = createMockReplyOperation();
|
||||
const abortForRestart = vi.spyOn(replyOperation, "abortForRestart");
|
||||
abortForRestart.mockImplementationOnce(() => {
|
||||
Object.defineProperty(replyOperation, "result", {
|
||||
value: { kind: "aborted", code: "aborted_for_restart" } as const,
|
||||
configurable: true,
|
||||
});
|
||||
return true;
|
||||
});
|
||||
state.runEmbeddedAgentMock.mockRejectedValueOnce(
|
||||
new Error("embedded runner failed", {
|
||||
cause: new SessionWriteLockStaleError({
|
||||
lockPath: "sqlite:session-write:agent:main:main",
|
||||
owner: "replacement gateway",
|
||||
staleReasons: ["lease-lost"],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
const confirmRestartRecoveryArmedAfterLeaseLoss = vi.fn(async () => true);
|
||||
|
||||
const executeAgentTurn = await getExecuteAgentTurnForTest();
|
||||
const result = await executeAgentTurn({
|
||||
...createMinimalRunAgentTurnParams({ replyOperation }),
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss,
|
||||
isRestartRecoveryArmed: () => false,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ kind: "final", payload: { text: SILENT_REPLY_TOKEN } });
|
||||
expect(confirmRestartRecoveryArmedAfterLeaseLoss).toHaveBeenCalledOnce();
|
||||
expect(abortForRestart).toHaveBeenCalledOnce();
|
||||
expect(failMock).not.toHaveBeenCalled();
|
||||
expect(
|
||||
emitAgentEvent.mock.calls.filter(
|
||||
([event]) =>
|
||||
event.stream === "lifecycle" &&
|
||||
event.data.phase === "end" &&
|
||||
event.data.stopReason === "restart",
|
||||
),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
emitAgentEvent.mock.calls.some(
|
||||
([event]) => event.stream === "lifecycle" && event.data.phase === "error",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves restart ownership when an aborted embedded runner resolves normally", async () => {
|
||||
const agentEvents = await import("../../infra/agent-events.js");
|
||||
const emitAgentEvent = vi.mocked(agentEvents.emitAgentEvent);
|
||||
|
||||
@@ -105,7 +105,6 @@ export type AgentTurnParams = {
|
||||
toolProgressDetail?: "explain" | "raw";
|
||||
replyMediaContext?: ReplyMediaContext;
|
||||
onCompactionNoticePayload?: (payload: ReplyPayload) => Promise<void> | void;
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss?: () => Promise<boolean>;
|
||||
isRestartRecoveryArmed?: () => boolean;
|
||||
};
|
||||
|
||||
|
||||
@@ -453,7 +453,6 @@ export async function runReplyAgent(
|
||||
beginBeforeAgentReply,
|
||||
checkpointBeforeAgentReply,
|
||||
clear: clearRestartRecoveryDeliveryClaim,
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss,
|
||||
isArmed: isRestartRecoveryArmed,
|
||||
} = createReplyAgentRestartRecoveryController({
|
||||
activeSessionStore,
|
||||
@@ -521,7 +520,6 @@ export async function runReplyAgent(
|
||||
blockStreamingEnabled,
|
||||
cfg,
|
||||
checkpointBeforeAgentReply,
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss,
|
||||
commandBody,
|
||||
defaultModel,
|
||||
followupRun,
|
||||
|
||||
@@ -5,7 +5,6 @@ import { join } from "node:path";
|
||||
import { createRequireRecord } from "openclaw/plugin-sdk/test-fixtures";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
|
||||
import { SessionWriteLockStaleError } from "../../agents/session-write-lock-error.js";
|
||||
import { clearRuntimeConfigSnapshot, setRuntimeConfigSnapshot } from "../../config/config.js";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import {
|
||||
@@ -23,7 +22,6 @@ import {
|
||||
import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js";
|
||||
import { createTestUserTurnTranscriptTarget } from "../../sessions/user-turn-transcript.test-support.js";
|
||||
import type { TemplateContext } from "../templating.js";
|
||||
import { SILENT_REPLY_TOKEN } from "../tokens.js";
|
||||
import {
|
||||
GENERIC_EXTERNAL_RUN_FAILURE_TEXT,
|
||||
HEARTBEAT_EXTERNAL_RUN_FAILURE_TEXT,
|
||||
@@ -2473,73 +2471,6 @@ describe("runReplyAgent pending final delivery capture", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves an adopted restart claim when the replacement takes the SQLite lease", async () => {
|
||||
const { sessionEntry, sessionStore, storePath } = await makeSessionFixture({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryDeliveryRequestFingerprint: "request-fingerprint",
|
||||
restartRecoveryDeliveryRunId: "msg",
|
||||
restartRecoveryDeliverySourceRunId: "control-ui-run",
|
||||
status: "running",
|
||||
});
|
||||
const replyOperation = createReplyOperation({
|
||||
sessionKey: "main",
|
||||
sessionId: "session",
|
||||
resetTriggered: false,
|
||||
});
|
||||
replyOperation.setPhase("running");
|
||||
state.runEmbeddedAgentMock.mockImplementationOnce(async () => {
|
||||
const current = await readStoredMainSession(storePath);
|
||||
expect(current.restartRecoveryDeliveryRunId).toBe("msg");
|
||||
await replaceSessionEntry(
|
||||
{ storePath, sessionKey: "main" },
|
||||
{
|
||||
...current,
|
||||
abortedLastRun: true,
|
||||
status: "killed",
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
);
|
||||
throw new SessionWriteLockStaleError({
|
||||
lockPath: "sqlite:session-write:agent:main:main",
|
||||
owner: "replacement gateway",
|
||||
staleReasons: ["lease-lost"],
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
const { run } = createMinimalRun({
|
||||
replyOperation,
|
||||
sessionCtx: {
|
||||
Provider: "webchat",
|
||||
OriginatingChannel: "webchat",
|
||||
},
|
||||
runOverrides: { agentId: "main", messageProvider: "webchat" },
|
||||
sessionEntry,
|
||||
sessionStore,
|
||||
sessionKey: "main",
|
||||
storePath,
|
||||
});
|
||||
|
||||
await expect(run()).resolves.toEqual({ text: SILENT_REPLY_TOKEN });
|
||||
|
||||
expect(replyOperation.result).toEqual({
|
||||
kind: "aborted",
|
||||
code: "aborted_for_restart",
|
||||
});
|
||||
expect(await readStoredMainSession(storePath)).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
restartRecoveryDeliveryRunId: "msg",
|
||||
restartRecoveryDeliverySourceRunId: "control-ui-run",
|
||||
status: "killed",
|
||||
});
|
||||
expect(
|
||||
(await readStoredMainSession(storePath)).restartRecoveryTerminalRunIds,
|
||||
).toBeUndefined();
|
||||
} finally {
|
||||
replyOperation.complete();
|
||||
}
|
||||
});
|
||||
|
||||
it("fires onAdopted after restart recovery delivery context persist completes", async () => {
|
||||
const { sessionEntry, sessionStore, storePath } = await makeSessionFixture();
|
||||
const expectedSourceTurnId = requireBuiltChannelSourceTurnId({
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
|
||||
import {
|
||||
@@ -11,7 +10,6 @@ import {
|
||||
replaceSessionEntry,
|
||||
updateSessionEntry,
|
||||
} from "../../config/sessions/session-accessor.js";
|
||||
import { resolveSqliteReadScope } from "../../config/sessions/session-accessor.sqlite-scope.js";
|
||||
import type { InternalSessionEntry, SessionEntry } from "../../config/sessions/types.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
|
||||
import { createUserTurnTranscriptRecorder } from "../../sessions/user-turn-transcript.js";
|
||||
@@ -19,7 +17,6 @@ import type {
|
||||
UserTurnTranscriptRecorder,
|
||||
UserTurnTranscriptTarget,
|
||||
} from "../../sessions/user-turn-transcript.types.js";
|
||||
import { openOpenClawAgentDatabase } from "../../state/openclaw-agent-db.js";
|
||||
import { createReplyRestartRecoveryClaimController } from "./restart-recovery-claim.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
@@ -45,24 +42,6 @@ function createTestAdmission(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function replaceSessionEntryFromIndependentConnection(params: {
|
||||
entry: SessionEntry;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
}): void {
|
||||
const database = openOpenClawAgentDatabase(
|
||||
resolveSqliteReadScope({ storePath: params.storePath, sessionKey: params.sessionKey }),
|
||||
);
|
||||
const external = new DatabaseSync(database.path);
|
||||
try {
|
||||
external
|
||||
.prepare("UPDATE session_nodes SET entry_json = ?, updated_at = ? WHERE session_key = ?")
|
||||
.run(JSON.stringify(params.entry), params.entry.updatedAt, params.sessionKey);
|
||||
} finally {
|
||||
external.close();
|
||||
}
|
||||
}
|
||||
|
||||
describe("createReplyRestartRecoveryClaimController", () => {
|
||||
it.each([
|
||||
{ receiptState: undefined, expectedStatus: "done" },
|
||||
@@ -458,120 +437,4 @@ describe("createReplyRestartRecoveryClaimController", () => {
|
||||
status: "done",
|
||||
});
|
||||
});
|
||||
|
||||
it("confirms an active replacement claim after SQLite lease loss", async () => {
|
||||
const root = tempDirs.make("openclaw-reply-restart-handoff-");
|
||||
const storePath = path.join(root, "sessions.json");
|
||||
const sessionKey = "agent:main:main";
|
||||
const sessionId = "session";
|
||||
let entry: SessionEntry = {
|
||||
abortedLastRun: false,
|
||||
restartRecoveryDeliveryRunId: "old-run",
|
||||
restartRecoveryDeliverySourceRunId: "source-run",
|
||||
sessionId,
|
||||
status: "running",
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
await replaceSessionEntry({ storePath, sessionKey }, entry);
|
||||
const controller = createReplyRestartRecoveryClaimController({
|
||||
admissionRunId: "old-run",
|
||||
getEntry: () => entry,
|
||||
getSessionId: () => sessionId,
|
||||
isRestartAbort: () => false,
|
||||
resolveDeliveryContext: () => undefined,
|
||||
sessionKey,
|
||||
setEntry: (next) => {
|
||||
entry = next;
|
||||
},
|
||||
storePath,
|
||||
});
|
||||
await expect(controller.admitUserTurn()).resolves.toBe("admitted");
|
||||
|
||||
replaceSessionEntryFromIndependentConnection({
|
||||
entry: {
|
||||
...entry,
|
||||
abortedLastRun: true,
|
||||
status: "killed",
|
||||
updatedAt: Date.now() + 1,
|
||||
},
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
|
||||
expect(entry.abortedLastRun).toBe(false);
|
||||
await expect(controller.confirmRestartRecoveryArmedAfterLeaseLoss()).resolves.toBe(true);
|
||||
expect(entry).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
restartRecoveryDeliveryRunId: "old-run",
|
||||
restartRecoveryDeliverySourceRunId: "source-run",
|
||||
status: "killed",
|
||||
});
|
||||
|
||||
await controller.clear();
|
||||
expect(loadSessionEntry({ storePath, sessionKey, readConsistency: "latest" })).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
restartRecoveryDeliveryRunId: "old-run",
|
||||
restartRecoveryDeliverySourceRunId: "source-run",
|
||||
status: "killed",
|
||||
});
|
||||
});
|
||||
|
||||
it("confirms a completed replacement handoff by its terminal source marker", async () => {
|
||||
const root = tempDirs.make("openclaw-reply-completed-handoff-");
|
||||
const storePath = path.join(root, "sessions.json");
|
||||
const sessionKey = "agent:main:main";
|
||||
const sessionId = "session";
|
||||
let entry: SessionEntry = {
|
||||
abortedLastRun: false,
|
||||
restartRecoveryDeliveryRunId: "old-run",
|
||||
restartRecoveryDeliverySourceRunId: "source-run",
|
||||
sessionId,
|
||||
status: "running",
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
await replaceSessionEntry({ storePath, sessionKey }, entry);
|
||||
const controller = createReplyRestartRecoveryClaimController({
|
||||
admissionRunId: "old-run",
|
||||
getEntry: () => entry,
|
||||
getSessionId: () => sessionId,
|
||||
isRestartAbort: () => false,
|
||||
resolveDeliveryContext: () => undefined,
|
||||
sessionKey,
|
||||
setEntry: (next) => {
|
||||
entry = next;
|
||||
},
|
||||
storePath,
|
||||
});
|
||||
await expect(controller.admitUserTurn()).resolves.toBe("admitted");
|
||||
|
||||
replaceSessionEntryFromIndependentConnection({
|
||||
entry: {
|
||||
...entry,
|
||||
abortedLastRun: false,
|
||||
restartRecoveryDeliveryRunId: undefined,
|
||||
restartRecoveryDeliverySourceRunId: undefined,
|
||||
restartRecoveryTerminalRunIds: ["source-run"],
|
||||
status: "done",
|
||||
updatedAt: Date.now() + 1,
|
||||
},
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
|
||||
expect(entry.restartRecoveryTerminalRunIds).toBeUndefined();
|
||||
await expect(controller.confirmRestartRecoveryArmedAfterLeaseLoss()).resolves.toBe(true);
|
||||
expect(entry).toMatchObject({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryTerminalRunIds: ["source-run"],
|
||||
status: "done",
|
||||
});
|
||||
expect(entry.restartRecoveryDeliveryRunId).toBeUndefined();
|
||||
expect(entry.restartRecoveryDeliverySourceRunId).toBeUndefined();
|
||||
|
||||
await controller.clear();
|
||||
const persisted = loadSessionEntry({ storePath, sessionKey, readConsistency: "latest" });
|
||||
expect(persisted?.restartRecoveryTerminalRunIds).toEqual(["source-run"]);
|
||||
expect(persisted?.restartRecoveryDeliveryRunId).toBeUndefined();
|
||||
expect(persisted?.restartRecoveryDeliverySourceRunId).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -34,7 +34,6 @@ type ReplyRestartRecoveryClaimController = {
|
||||
};
|
||||
}) => Promise<void>;
|
||||
clear: () => Promise<void>;
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss: () => Promise<boolean>;
|
||||
isArmed: () => boolean;
|
||||
};
|
||||
|
||||
@@ -130,7 +129,6 @@ export function createReplyRestartRecoveryClaimController(params: {
|
||||
let recoveryRunId: string = randomUUID();
|
||||
let recoverySourceRunId: string | undefined;
|
||||
let tracked = false;
|
||||
let leaseLossRestartHandoffConfirmed = false;
|
||||
|
||||
const persistAdmissionPatch = async (options: {
|
||||
entry: SessionEntry;
|
||||
@@ -438,43 +436,8 @@ export function createReplyRestartRecoveryClaimController(params: {
|
||||
return true;
|
||||
};
|
||||
|
||||
const confirmRestartRecoveryArmedAfterLeaseLoss = async (): Promise<boolean> => {
|
||||
if (!tracked || !params.sessionKey || !params.storePath || !recoverySourceRunId) {
|
||||
return false;
|
||||
}
|
||||
// Lease loss means the replacement may have advanced the authoritative row
|
||||
// past this process's cache. This cold error path performs one latest read.
|
||||
const persisted = loadSessionEntry({
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
clone: false,
|
||||
hydrateSkillPromptRefs: false,
|
||||
readConsistency: "latest",
|
||||
});
|
||||
if (!persisted || persisted.sessionId !== params.getSessionId()) {
|
||||
return false;
|
||||
}
|
||||
params.setEntry(persisted);
|
||||
const activeHandoff =
|
||||
persisted.abortedLastRun === true &&
|
||||
normalizeOptionalString(persisted.restartRecoveryDeliveryRunId) === recoveryRunId &&
|
||||
hasRestartRecoverySourceClaim(persisted, recoverySourceRunId);
|
||||
// The replacement may finish and clear the active claim before the old
|
||||
// owner observes lease loss. Its terminal source marker proves completion.
|
||||
const completedHandoff = hasRestartRecoveryTerminalRun(persisted, recoverySourceRunId);
|
||||
const armed = activeHandoff || completedHandoff;
|
||||
leaseLossRestartHandoffConfirmed ||= armed;
|
||||
return armed;
|
||||
};
|
||||
|
||||
const clear = async (): Promise<void> => {
|
||||
if (
|
||||
!tracked ||
|
||||
!params.sessionKey ||
|
||||
!params.storePath ||
|
||||
params.isRestartAbort() ||
|
||||
leaseLossRestartHandoffConfirmed
|
||||
) {
|
||||
if (!tracked || !params.sessionKey || !params.storePath || params.isRestartAbort()) {
|
||||
return;
|
||||
}
|
||||
const persisted = await updateSessionEntry(
|
||||
@@ -568,7 +531,6 @@ export function createReplyRestartRecoveryClaimController(params: {
|
||||
beginBeforeAgentReply,
|
||||
checkpointBeforeAgentReply,
|
||||
clear,
|
||||
confirmRestartRecoveryArmedAfterLeaseLoss,
|
||||
isArmed,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -55,27 +55,6 @@ vi.mock("../../plugin-sdk/browser-maintenance.js", () => ({
|
||||
closeTrackedBrowserTabsForSessions: sessionCleanupMocks.closeTrackedBrowserTabsForSessions,
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/session-write-lock.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../../agents/session-write-lock.js")>(
|
||||
"../../agents/session-write-lock.js",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => {} })),
|
||||
resolveSessionLockMaxHoldFromTimeout: vi.fn(
|
||||
({
|
||||
timeoutMs,
|
||||
graceMs = 2 * 60 * 1000,
|
||||
minMs = 5 * 60 * 1000,
|
||||
}: {
|
||||
timeoutMs: number;
|
||||
graceMs?: number;
|
||||
minMs?: number;
|
||||
}) => Math.max(minMs, timeoutMs + graceMs),
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
const suiteTempDirs = createSuiteTempRootTracker({ prefix: "openclaw-session-hooks-" });
|
||||
|
||||
async function createStorePath(prefix: string): Promise<string> {
|
||||
|
||||
@@ -111,28 +111,6 @@ vi.mock("../../infra/channel-summary.js", () => ({
|
||||
buildChannelSummary: channelSummaryMocks.buildChannelSummary,
|
||||
}));
|
||||
|
||||
// Perf: session-store locks are exercised elsewhere; most session tests don't need FS lock files.
|
||||
vi.mock("../../agents/session-write-lock.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../../agents/session-write-lock.js")>(
|
||||
"../../agents/session-write-lock.js",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
acquireSessionWriteLock: vi.fn(async () => ({ release: async () => {} })),
|
||||
resolveSessionLockMaxHoldFromTimeout: vi.fn(
|
||||
({
|
||||
timeoutMs,
|
||||
graceMs = 2 * 60 * 1000,
|
||||
minMs = 5 * 60 * 1000,
|
||||
}: {
|
||||
timeoutMs: number;
|
||||
graceMs?: number;
|
||||
minMs?: number;
|
||||
}) => Math.max(minMs, timeoutMs + graceMs),
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../../agents/prepared-model-catalog.js", () => ({
|
||||
loadProviderScopedThinkingCatalog: vi.fn(async () => []),
|
||||
loadPreparedModelCatalog: vi.fn(async () => [
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user