From c84a59284b15b8600a456768d576228ce4cd3829 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 19 Jul 2026 00:13:19 -0700 Subject: [PATCH] fix(android): stop push-to-talk from dropping speech after mid-hold pauses (#110995) * fix(android): stop push-to-talk from dropping speech after mid-hold pauses The SpeechRecognizer PTT path ran one session with a 2.5s silence window, kept only the last final, and never restarted during a hold - speech after a mid-hold pause was silently dropped. Capture now uses an ordered rung ladder: API 33+ app-owned AudioRecord piped via EXTRA_AUDIO_SOURCE with EXTRA_SEGMENTED_SESSION, then silence-keyed segmented sessions, then a restart-during-hold single-session fallback for older devices; failed holds only descend. Final segments accumulate and merge locale-aware (no ASCII separators after CJK/Arabic punctuation), with a trailing live partial deduped against the last final. Release waits one bounded grace for the terminal recognizer callback so late finals are not truncated; rapid re-press drains the prior release instead of cancelling it; cancellation mid-release still tears capture down, resumes realtime talk, and resolves the stop payload. Raw-rung release also stops the recognizer since EXTRA_AUDIO_SOURCE is optional and a service may run its own microphone. Input level meters from our PCM stream in the raw rung where onRmsChanged never fires. * fix(android): API-33 recognizer extras behind RequiresApi and inventory refresh Min-SDK lint (InlinedApi) flagged the API 33 RecognizerIntent extras; they now live in RequiresApi(TIRAMISU) helpers behind explicit SDK gates, and the native i18n inventory is refreshed for the moved lines. * chore(android): rebuild native locale artifacts for inventory order --- apps/.i18n/native-source.json | 66 +- apps/.i18n/native/ar.json | 10 +- apps/.i18n/native/de.json | 10 +- apps/.i18n/native/es.json | 10 +- apps/.i18n/native/fa.json | 10 +- apps/.i18n/native/fr.json | 10 +- apps/.i18n/native/hi.json | 10 +- apps/.i18n/native/id.json | 10 +- apps/.i18n/native/it.json | 10 +- apps/.i18n/native/ja-JP.json | 10 +- apps/.i18n/native/ko.json | 10 +- apps/.i18n/native/nl.json | 10 +- apps/.i18n/native/pl.json | 10 +- apps/.i18n/native/pt-BR.json | 10 +- apps/.i18n/native/ru.json | 10 +- apps/.i18n/native/sv.json | 10 +- apps/.i18n/native/th.json | 10 +- apps/.i18n/native/tr.json | 10 +- apps/.i18n/native/uk.json | 10 +- apps/.i18n/native/vi.json | 10 +- apps/.i18n/native/zh-CN.json | 10 +- apps/.i18n/native/zh-TW.json | 10 +- .../app/voice/PushToTalkRecognitionLadder.kt | 25 + .../app/voice/PushToTalkTranscriptMerger.kt | 39 ++ .../ai/openclaw/app/voice/TalkModeManager.kt | 594 ++++++++++++++++-- .../voice/PushToTalkRecognitionLadderTest.kt | 56 ++ .../voice/PushToTalkTranscriptMergerTest.kt | 90 +++ .../openclaw/app/voice/TalkModeManagerTest.kt | 159 +++++ 28 files changed, 1045 insertions(+), 194 deletions(-) create mode 100644 apps/android/app/src/main/java/ai/openclaw/app/voice/PushToTalkRecognitionLadder.kt create mode 100644 apps/android/app/src/main/java/ai/openclaw/app/voice/PushToTalkTranscriptMerger.kt create mode 100644 apps/android/app/src/test/java/ai/openclaw/app/voice/PushToTalkRecognitionLadderTest.kt create mode 100644 apps/android/app/src/test/java/ai/openclaw/app/voice/PushToTalkTranscriptMergerTest.kt diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index b2e584705521..c78cd5a6cac3 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -13043,7 +13043,7 @@ }, { "kind": "ui-call", - "line": 532, + "line": 602, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Speech recognizer unavailable", "surface": "android", @@ -13051,15 +13051,7 @@ }, { "kind": "ui-call", - "line": 596, - "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", - "source": "Listening (PTT)", - "surface": "android", - "id": "native.android.0faf9df3e5f708df" - }, - { - "kind": "ui-call", - "line": 693, + "line": 800, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Ready", "surface": "android", @@ -13067,7 +13059,7 @@ }, { "kind": "ui-call", - "line": 911, + "line": 1018, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Start failed: $message", "surface": "android", @@ -13075,7 +13067,7 @@ }, { "kind": "ui-call", - "line": 1001, + "line": 1114, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Connecting…", "surface": "android", @@ -13083,7 +13075,7 @@ }, { "kind": "ui-call", - "line": 1070, + "line": 1183, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Off", "surface": "android", @@ -13091,7 +13083,7 @@ }, { "kind": "ui-call", - "line": 1073, + "line": 1186, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Talk failed: Realtime provider closed unexpectedly.", "surface": "android", @@ -13099,7 +13091,7 @@ }, { "kind": "ui-call", - "line": 1078, + "line": 1191, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Talk failed: Realtime provider closed: $reason", "surface": "android", @@ -13107,7 +13099,7 @@ }, { "kind": "ui-call", - "line": 2216, + "line": 2546, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Thinking…", "surface": "android", @@ -13115,7 +13107,7 @@ }, { "kind": "ui-call", - "line": 2233, + "line": 2563, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Gateway not connected", "surface": "android", @@ -13123,7 +13115,7 @@ }, { "kind": "ui-call", - "line": 2246, + "line": 2576, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Aborted", "surface": "android", @@ -13131,7 +13123,7 @@ }, { "kind": "ui-call", - "line": 2246, + "line": 2576, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Chat error", "surface": "android", @@ -13139,7 +13131,7 @@ }, { "kind": "ui-call", - "line": 2263, + "line": 2593, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "No reply", "surface": "android", @@ -13147,7 +13139,7 @@ }, { "kind": "ui-call", - "line": 2279, + "line": 2609, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Talk failed: $message", "surface": "android", @@ -13155,7 +13147,7 @@ }, { "kind": "ui-call", - "line": 2504, + "line": 2902, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Generating voice…", "surface": "android", @@ -13163,7 +13155,7 @@ }, { "kind": "ui-call", - "line": 2532, + "line": 2930, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Speak failed: $message", "surface": "android", @@ -13171,7 +13163,7 @@ }, { "kind": "ui-call", - "line": 2681, + "line": 3079, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Speaking…", "surface": "android", @@ -13179,7 +13171,15 @@ }, { "kind": "ui-call", - "line": 2998, + "line": 3430, + "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", + "source": "Listening (PTT)", + "surface": "android", + "id": "native.android.0faf9df3e5f708df" + }, + { + "kind": "ui-call", + "line": 3441, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Microphone permission required", "surface": "android", @@ -13187,7 +13187,7 @@ }, { "kind": "ui-call", - "line": 3004, + "line": 3447, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Audio error", "surface": "android", @@ -13195,7 +13195,7 @@ }, { "kind": "ui-call", - "line": 3005, + "line": 3448, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Client error", "surface": "android", @@ -13203,7 +13203,7 @@ }, { "kind": "ui-call", - "line": 3006, + "line": 3449, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Network error", "surface": "android", @@ -13211,7 +13211,7 @@ }, { "kind": "ui-call", - "line": 3007, + "line": 3450, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Network timeout", "surface": "android", @@ -13219,7 +13219,7 @@ }, { "kind": "ui-call", - "line": 3009, + "line": 3452, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Recognizer busy", "surface": "android", @@ -13227,7 +13227,7 @@ }, { "kind": "ui-call", - "line": 3010, + "line": 3453, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Server error", "surface": "android", @@ -13235,7 +13235,7 @@ }, { "kind": "ui-call", - "line": 3011, + "line": 3454, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Listening", "surface": "android", @@ -13243,7 +13243,7 @@ }, { "kind": "ui-call", - "line": 3012, + "line": 3455, "path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt", "source": "Speech error ($error)", "surface": "android", diff --git a/apps/.i18n/native/ar.json b/apps/.i18n/native/ar.json index 873d2d0fa0b3..c6dc48dddfac 100644 --- a/apps/.i18n/native/ar.json +++ b/apps/.i18n/native/ar.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "أداة التعرّف على الكلام غير متاحة" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "جارٍ الاستماع (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "جارٍ التحدث…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "جارٍ الاستماع (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/de.json b/apps/.i18n/native/de.json index 68225e2a0ccd..50875c4fe719 100644 --- a/apps/.i18n/native/de.json +++ b/apps/.i18n/native/de.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Spracherkennung nicht verfügbar" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Hört zu (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Spricht…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Hört zu (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/es.json b/apps/.i18n/native/es.json index b33175f008df..794c89bf9a9f 100644 --- a/apps/.i18n/native/es.json +++ b/apps/.i18n/native/es.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "El reconocimiento de voz no está disponible" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Escuchando (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Hablando…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Escuchando (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/fa.json b/apps/.i18n/native/fa.json index fa49ce1bc1ca..4fd76cdb8657 100644 --- a/apps/.i18n/native/fa.json +++ b/apps/.i18n/native/fa.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "تشخیص‌دهنده گفتار در دسترس نیست" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "در حال شنیدن (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "در حال صحبت…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "در حال شنیدن (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/fr.json b/apps/.i18n/native/fr.json index b6c604093d7b..dfb62676e3a9 100644 --- a/apps/.i18n/native/fr.json +++ b/apps/.i18n/native/fr.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Reconnaissance vocale indisponible" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Écoute (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Lecture en cours…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Écoute (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/hi.json b/apps/.i18n/native/hi.json index 532ef264522c..5e9e35401296 100644 --- a/apps/.i18n/native/hi.json +++ b/apps/.i18n/native/hi.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "वाक् पहचानकर्ता उपलब्ध नहीं है" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "सुन रहा है (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "बोल रहा है…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "सुन रहा है (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/id.json b/apps/.i18n/native/id.json index 1a0eeccde57b..357af8e9c273 100644 --- a/apps/.i18n/native/id.json +++ b/apps/.i18n/native/id.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Pengenal ucapan tidak tersedia" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Mendengarkan (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Berbicara…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Mendengarkan (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/it.json b/apps/.i18n/native/it.json index fa1b5765ca43..f359afefb2cb 100644 --- a/apps/.i18n/native/it.json +++ b/apps/.i18n/native/it.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Riconoscimento vocale non disponibile" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "In ascolto (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Riproduzione vocale…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "In ascolto (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/ja-JP.json b/apps/.i18n/native/ja-JP.json index 2ebcc5512565..02ff0b7a46dc 100644 --- a/apps/.i18n/native/ja-JP.json +++ b/apps/.i18n/native/ja-JP.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "音声認識を利用できません" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "聞き取り中（PTT）" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "読み上げ中…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "聞き取り中（PTT）" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/ko.json b/apps/.i18n/native/ko.json index bab741b20633..b959b910d439 100644 --- a/apps/.i18n/native/ko.json +++ b/apps/.i18n/native/ko.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "음성 인식기를 사용할 수 없습니다" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "듣는 중(PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "말하는 중…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "듣는 중(PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/nl.json b/apps/.i18n/native/nl.json index bbc7c296b2b5..e374d586f712 100644 --- a/apps/.i18n/native/nl.json +++ b/apps/.i18n/native/nl.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Spraakherkenning niet beschikbaar" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Luisteren (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Spreken…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Luisteren (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/pl.json b/apps/.i18n/native/pl.json index b72371f15ca4..b26d19db2a29 100644 --- a/apps/.i18n/native/pl.json +++ b/apps/.i18n/native/pl.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Rozpoznawanie mowy jest niedostępne" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Nasłuchiwanie (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Mówienie…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Nasłuchiwanie (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/pt-BR.json b/apps/.i18n/native/pt-BR.json index dbd0fafd448b..b562b6e8c35b 100644 --- a/apps/.i18n/native/pt-BR.json +++ b/apps/.i18n/native/pt-BR.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Reconhecimento de fala indisponível" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Ouvindo (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Falando…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Ouvindo (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/ru.json b/apps/.i18n/native/ru.json index 78a8e9b88118..e8e7b11c3d15 100644 --- a/apps/.i18n/native/ru.json +++ b/apps/.i18n/native/ru.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Распознавание речи недоступно" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Прослушивание (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Воспроизведение речи…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Прослушивание (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/sv.json b/apps/.i18n/native/sv.json index 1e5b9a76074b..780e5a4b7e58 100644 --- a/apps/.i18n/native/sv.json +++ b/apps/.i18n/native/sv.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Taligenkänning är inte tillgänglig" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Lyssnar (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Talar…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Lyssnar (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/th.json b/apps/.i18n/native/th.json index c212785834f7..1804842d25d9 100644 --- a/apps/.i18n/native/th.json +++ b/apps/.i18n/native/th.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "ไม่สามารถใช้ตัวจดจำเสียงพูดได้" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "กำลังฟัง (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "กำลังพูด…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "กำลังฟัง (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/tr.json b/apps/.i18n/native/tr.json index a0846efacdc6..af4ddaa6ed0b 100644 --- a/apps/.i18n/native/tr.json +++ b/apps/.i18n/native/tr.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Konuşma tanıyıcı kullanılamıyor" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Dinleniyor (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Konuşuyor…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Dinleniyor (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/uk.json b/apps/.i18n/native/uk.json index 20131bb1c126..274e7c3e9544 100644 --- a/apps/.i18n/native/uk.json +++ b/apps/.i18n/native/uk.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Розпізнавання мовлення недоступне" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Прослуховування (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Озвучення…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Прослуховування (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/vi.json b/apps/.i18n/native/vi.json index 08bfed4f3a0b..085a63205bc1 100644 --- a/apps/.i18n/native/vi.json +++ b/apps/.i18n/native/vi.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "Trình nhận dạng giọng nói không khả dụng" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "Đang nghe (PTT)" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "Đang nói…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "Đang nghe (PTT)" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/zh-CN.json b/apps/.i18n/native/zh-CN.json index 3e1f1986d118..34a174cb1242 100644 --- a/apps/.i18n/native/zh-CN.json +++ b/apps/.i18n/native/zh-CN.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "语音识别器不可用" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "正在聆听（按住说话）" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "正在朗读…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "正在聆听（按住说话）" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/.i18n/native/zh-TW.json b/apps/.i18n/native/zh-TW.json index e3775e822767..b0f2cb43956c 100644 --- a/apps/.i18n/native/zh-TW.json +++ b/apps/.i18n/native/zh-TW.json @@ -8158,11 +8158,6 @@ "source": "Speech recognizer unavailable", "translated": "語音辨識器無法使用" }, - { - "id": "native.android.0faf9df3e5f708df", - "source": "Listening (PTT)", - "translated": "正在聆聽（PTT）" - }, { "id": "native.android.47a59f8798682ea0", "source": "Ready", @@ -8238,6 +8233,11 @@ "source": "Speaking…", "translated": "正在朗讀…" }, + { + "id": "native.android.0faf9df3e5f708df", + "source": "Listening (PTT)", + "translated": "正在聆聽（PTT）" + }, { "id": "native.android.d1a7db2c16fd6d09", "source": "Microphone permission required", diff --git a/apps/android/app/src/main/java/ai/openclaw/app/voice/PushToTalkRecognitionLadder.kt b/apps/android/app/src/main/java/ai/openclaw/app/voice/PushToTalkRecognitionLadder.kt new file mode 100644 index 000000000000..25b2648464dd --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/app/voice/PushToTalkRecognitionLadder.kt @@ -0,0 +1,25 @@ +package ai.openclaw.app.voice + +internal enum class PushToTalkRecognitionCandidate { + RawAudioSegmented, + SilenceSegmented, + RestartingSingleSession, +} + +internal fun pushToTalkRecognitionCandidates( + supportsSegmentedRecognition: Boolean, + first: PushToTalkRecognitionCandidate?, +): List { + val available = + if (supportsSegmentedRecognition) { + PushToTalkRecognitionCandidate.entries + } else { + listOf(PushToTalkRecognitionCandidate.RestartingSingleSession) + } + if (first == null) return available + return available.dropWhile { it != first }.ifEmpty { listOf(PushToTalkRecognitionCandidate.RestartingSingleSession) } +} + +internal fun shouldAdvancePushToTalkRungAfterSegmentedSession( + candidate: PushToTalkRecognitionCandidate, +): Boolean = candidate == PushToTalkRecognitionCandidate.RawAudioSegmented diff --git a/apps/android/app/src/main/java/ai/openclaw/app/voice/PushToTalkTranscriptMerger.kt b/apps/android/app/src/main/java/ai/openclaw/app/voice/PushToTalkTranscriptMerger.kt new file mode 100644 index 000000000000..38dfe0f460ba --- /dev/null +++ b/apps/android/app/src/main/java/ai/openclaw/app/voice/PushToTalkTranscriptMerger.kt @@ -0,0 +1,39 @@ +package ai.openclaw.app.voice + +import java.util.Locale + +internal object PushToTalkTranscriptMerger { + private val trailingClosers = setOf('"', '\'', '’', '”', ')', ']', '}') + + fun merge( + finalSegments: List, + livePartial: String?, + ): String { + val segments = finalSegments.map { it.trim() }.filter { it.isNotEmpty() }.toMutableList() + val partial = livePartial?.trim().orEmpty() + val lastFinal = segments.lastOrNull() + if (partial.isNotEmpty() && (lastFinal == null || normalize(partial) != normalize(lastFinal))) { + segments += partial + } + + return buildString { + var previousEndsSentence = false + segments.forEachIndexed { index, segment -> + if (index > 0) { + append(if (previousEndsSentence) " " else ". ") + } + append(segment) + // Recognizers emit locale punctuation (。？؟…); injecting ASCII ". " after those corrupts + // the transcript. Separate with ". " only when a segment ends in a letter or digit. + val finalCharacter = segment.trimEnd { it in trailingClosers }.lastOrNull() + previousEndsSentence = finalCharacter != null && !finalCharacter.isLetterOrDigit() + } + } + } + + private fun normalize(value: String): String = + value + .trim() + .lowercase(Locale.ROOT) + .replace(Regex("""\s+"""), " ") +} diff --git a/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt b/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt index ffb7820962e3..ce2bd1a8deab 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt @@ -17,10 +17,14 @@ import android.media.AudioAttributes import android.media.AudioFocusRequest import android.media.AudioFormat import android.media.AudioManager +import android.media.AudioRecord import android.media.AudioTrack +import android.media.MediaRecorder +import android.os.Build import android.os.Bundle import android.os.Handler import android.os.Looper +import android.os.ParcelFileDescriptor import android.os.SystemClock import android.speech.RecognitionListener import android.speech.RecognizerIntent @@ -29,6 +33,7 @@ import android.speech.tts.TextToSpeech import android.speech.tts.UtteranceProgressListener import android.util.Base64 import android.util.Log +import androidx.annotation.RequiresApi import androidx.core.content.ContextCompat import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CompletableDeferred @@ -53,15 +58,19 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeout +import kotlinx.coroutines.withTimeoutOrNull +import kotlinx.coroutines.yield import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.buildJsonObject +import java.io.IOException import java.util.LinkedHashMap import java.util.Locale import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicLong import kotlin.coroutines.coroutineContext @@ -183,6 +192,57 @@ private data class PendingRealtimePlaybackMark( var targetFrame: Long? = null, ) +private class PushToTalkAudioSource( + val readDescriptor: ParcelFileDescriptor, + private val writeStream: ParcelFileDescriptor.AutoCloseOutputStream, + private val audioRecord: AudioRecord, +) { + private val finishRequested = AtomicBoolean(false) + private val inputFinished = AtomicBoolean(false) + private val descriptorClosed = AtomicBoolean(false) + var pumpJob: Job? = null + + fun requestFinish() { + if (!finishRequested.compareAndSet(false, true)) return + runCatching { audioRecord.stop() } + } + + fun finishFromPump() { + if (!inputFinished.compareAndSet(false, true)) return + runCatching { audioRecord.stop() } + runCatching { audioRecord.release() } + runCatching { writeStream.close() } + } + + fun close() { + requestFinish() + pumpJob?.cancel() + finishFromPump() + pumpJob = null + if (descriptorClosed.compareAndSet(false, true)) { + runCatching { readDescriptor.close() } + } + } +} + +private sealed interface PushToTalkRecognitionRung { + val candidate: PushToTalkRecognitionCandidate + + data class RawAudioSegmented( + val source: PushToTalkAudioSource, + ) : PushToTalkRecognitionRung { + override val candidate = PushToTalkRecognitionCandidate.RawAudioSegmented + } + + data object SilenceSegmented : PushToTalkRecognitionRung { + override val candidate = PushToTalkRecognitionCandidate.SilenceSegmented + } + + data object RestartingSingleSession : PushToTalkRecognitionRung { + override val candidate = PushToTalkRecognitionCandidate.RestartingSingleSession + } +} + class TalkModeManager internal constructor( private val context: Context, private val scope: CoroutineScope, @@ -209,6 +269,10 @@ class TalkModeManager internal constructor( private const val maxConversationEntries = 40 private const val realtimePlaybackBufferMs = 240 private const val realtimeUserFinalRewriteGraceMs = 1_500L + private const val pushToTalkSampleRateHz = 16_000 + private const val pushToTalkReleaseGraceMs = 5_000L + private const val pushToTalkReleaseDrainTimeoutMs = 6_000L + private const val pushToTalkRestartDelayMs = 200L } private val mainHandler = Handler(Looper.getMainLooper()) @@ -288,6 +352,10 @@ class TalkModeManager internal constructor( private var pttAutoStopEnabled = false private var pttTimeoutJob: Job? = null private var pttCompletion: CompletableDeferred? = null + private var pttRecognitionRung: PushToTalkRecognitionRung? = null + private var pttReleaseCompletion: CompletableDeferred? = null + private val pttFinalSegments = mutableListOf() + private var pttLivePartial = "" private var silenceJob: Job? = null private var silenceWindowMs = TalkDefaults.defaultSilenceTimeoutMs @@ -511,7 +579,9 @@ class TalkModeManager internal constructor( } // PTT begin is idempotent so gateway retries don't start multiple recognizers. activePttCaptureId?.let { - return PushToTalkStartResult.Existing(TalkPttStartPayload(captureId = it)) + if (pttReleaseCompletion == null) { + return PushToTalkStartResult.Existing(TalkPttStartPayload(captureId = it)) + } } finishingPttCaptureId?.let { throw IllegalStateException("PTT_BUSY: previous push-to-talk turn is still finishing") @@ -537,8 +607,14 @@ class TalkModeManager internal constructor( val captureGeneration = startGeneration.get() return try { withContext(Dispatchers.Main) { + val hasPendingRelease = pttReleaseCompletion != null + if (hasPendingRelease) { + drainPushToTalkReleaseBeforeBegin() + } activePttCaptureId?.let { - return@withContext PushToTalkStartResult.Existing(TalkPttStartPayload(captureId = it)) + if (!hasPendingRelease) { + return@withContext PushToTalkStartResult.Existing(TalkPttStartPayload(captureId = it)) + } } finishingPttCaptureId?.let { throw IllegalStateException("PTT_BUSY: previous push-to-talk turn is still finishing") @@ -559,6 +635,10 @@ class TalkModeManager internal constructor( recognizer?.cancel() recognizer?.destroy() recognizer = null + closePushToTalkRung() + pttReleaseCompletion = null + pttFinalSegments.clear() + pttLivePartial = "" lastTranscript = "" lastHeardAtMs = null activePttCaptureId = captureId @@ -577,9 +657,13 @@ class TalkModeManager internal constructor( ) { throw IllegalStateException("NODE_BACKGROUND_UNAVAILABLE: command requires foreground") } - recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) } - startListeningInternal(markListening = true) + recognizer = + SpeechRecognizer.createSpeechRecognizer(context).also { + it.setRecognitionListener(recognitionListener(captureId)) + } + startPushToTalkRecognition(captureId) } catch (err: Throwable) { + closePushToTalkRung() runCatching { recognizer?.cancel() } runCatching { recognizer?.destroy() } recognizer = null @@ -624,59 +708,82 @@ class TalkModeManager internal constructor( } internal suspend fun endPushToTalk(captureId: String): TalkPttStopPayload = - withContext(Dispatchers.Main) { - val cleared = - clearPushToTalkRecognition(captureId) - ?: return@withContext TalkPttStopPayload(captureId = captureId, transcript = null, status = "idle") - val transcript = cleared.transcript + try { + withContext(Dispatchers.Main) { + awaitPushToTalkRelease(captureId) + val cleared = + clearPushToTalkRecognition(captureId) + ?: return@withContext TalkPttStopPayload(captureId = captureId, transcript = null, status = "idle") + val transcript = cleared.transcript - if (transcript.isEmpty()) { - setStatus(if (_isEnabled.value) nativeText("Listening") else nativeText("Ready")) - resumeRealtimeCaptureAfterPushToTalk(captureId) - return@withContext finishPushToTalk( - TalkPttStopPayload(captureId = captureId, transcript = null, status = "empty"), - cleared.completion, - ) - } + if (transcript.isEmpty()) { + setStatus(if (_isEnabled.value) nativeText("Listening") else nativeText("Ready")) + resumeRealtimeCaptureAfterPushToTalk(captureId) + return@withContext finishPushToTalk( + TalkPttStopPayload(captureId = captureId, transcript = null, status = "empty"), + cleared.completion, + ) + } - if (!isConnected()) { - setStatus(nativeText("Gateway not connected")) - resumeRealtimeCaptureAfterPushToTalk(captureId) - return@withContext finishPushToTalk( - TalkPttStopPayload(captureId = captureId, transcript = transcript, status = "offline"), - cleared.completion, - ) - } + if (!isConnected()) { + setStatus(nativeText("Gateway not connected")) + resumeRealtimeCaptureAfterPushToTalk(captureId) + return@withContext finishPushToTalk( + TalkPttStopPayload(captureId = captureId, transcript = transcript, status = "offline"), + cleared.completion, + ) + } - setStatus(nativeText("Thinking…"), awaitingAgent = true) - lateinit var finishingJob: Job - finishingJob = - // Gateway-scoped so a switch drops the stale finalize; the NonCancellable - // finally still resumes capture when the scope cancels this job. - gatewayWorkScope.launch(start = CoroutineStart.LAZY) { - try { - finalizeTranscript(transcript) - } finally { - withContext(NonCancellable + Dispatchers.Main) { - resumeRealtimeCaptureAfterPushToTalk(captureId) - clearFinishingPushToTalk(captureId, finishingJob) + setStatus(nativeText("Thinking…"), awaitingAgent = true) + lateinit var finishingJob: Job + finishingJob = + // Gateway-scoped so a switch drops the stale finalize; the NonCancellable + // finally still resumes capture when the scope cancels this job. + gatewayWorkScope.launch(start = CoroutineStart.LAZY) { + try { + finalizeTranscript(transcript) + } finally { + withContext(NonCancellable + Dispatchers.Main) { + resumeRealtimeCaptureAfterPushToTalk(captureId) + clearFinishingPushToTalk(captureId, finishingJob) + } } } + // Cancellation can win before a lazy coroutine enters its body, in which + // case its finally block never runs. Completion still releases ownership. + finishingJob.invokeOnCompletion { clearFinishingPushToTalk(captureId, finishingJob) } + // Publish the job before it can run so stop() cannot clear ownership while + // an untracked finalizer still uses shared chat and playback state. + synchronized(finishingPttLock) { + finishingPttCaptureId = captureId + finishingPttJob = finishingJob + finishingJob.start() } - // Cancellation can win before a lazy coroutine enters its body, in which - // case its finally block never runs. Completion still releases ownership. - finishingJob.invokeOnCompletion { clearFinishingPushToTalk(captureId, finishingJob) } - // Publish the job before it can run so stop() cannot clear ownership while - // an untracked finalizer still uses shared chat and playback state. - synchronized(finishingPttLock) { - finishingPttCaptureId = captureId - finishingPttJob = finishingJob - finishingJob.start() + finishPushToTalk( + TalkPttStopPayload(captureId = captureId, transcript = transcript, status = "queued"), + cleared.completion, + ) } - finishPushToTalk( - TalkPttStopPayload(captureId = captureId, transcript = transcript, status = "queued"), - cleared.completion, - ) + } catch (err: CancellationException) { + // Mirror the normal termination tail: resume realtime capture, restore status, and + // resolve the PTT completion so a cancelled end (gateway drop) cannot leave Talk + // paused or an awaiter stuck behind the release wait. + withContext(NonCancellable + Dispatchers.Main) { + val cleared = clearPushToTalkRecognition(captureId) + if (cleared != null) { + setStatus(if (_isEnabled.value) nativeText("Listening") else nativeText("Ready")) + resumeRealtimeCaptureAfterPushToTalk(captureId) + finishPushToTalk( + TalkPttStopPayload( + captureId = captureId, + transcript = cleared.transcript.ifEmpty { null }, + status = "cancelled", + ), + cleared.completion, + ) + } + } + throw err } /** Cancels push-to-talk capture without sending the current transcript. */ @@ -934,9 +1041,15 @@ class TalkModeManager internal constructor( restartJob = null silenceJob?.cancel() silenceJob = null + closePushToTalkRung() + pttReleaseCompletion?.cancel() + pttReleaseCompletion = null + pttFinalSegments.clear() + pttLivePartial = "" lastTranscript = "" lastHeardAtMs = null _isListening.value = false + _inputLevel.value = 0f setStatus(nativeText("Off"), state = TalkStatusState.Off) stopRealtimeRelay() stopSpeaking() @@ -2105,6 +2218,205 @@ class TalkModeManager internal constructor( private val transcriptSpaceAfterPunctuation = setOf('.', '!', '?', ',', ':', ';', ')', ']', '}', '"', '\'', '’', '”') + // API 33 adds segmented callbacks and caller-owned audio. Keep this ordered ladder + // in one place: removing the restart rung makes older devices drop speech after a pause. + private fun pushToTalkCandidates(first: PushToTalkRecognitionCandidate?): List = + pushToTalkRecognitionCandidates( + supportsSegmentedRecognition = Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU, + first = first, + ) + + private fun startPushToTalkRecognition( + captureId: String, + firstCandidate: PushToTalkRecognitionCandidate? = null, + ) { + val recognizerInstance = recognizer ?: error("Speech recognizer unavailable") + var lastFailure: Throwable? = null + for (candidate in pushToTalkCandidates(firstCandidate)) { + try { + val rung = + when (candidate) { + PushToTalkRecognitionCandidate.RawAudioSegmented -> + PushToTalkRecognitionRung.RawAudioSegmented(openPushToTalkAudioSource()) + PushToTalkRecognitionCandidate.SilenceSegmented -> PushToTalkRecognitionRung.SilenceSegmented + PushToTalkRecognitionCandidate.RestartingSingleSession -> + PushToTalkRecognitionRung.RestartingSingleSession + } + pttRecognitionRung = rung + recognizerInstance.startListening(pushToTalkRecognizerIntent(rung)) + _isListening.value = true + setStatus(nativeText("Listening (PTT)")) + return + } catch (err: Throwable) { + lastFailure = err + closePushToTalkRung() + Log.w(tag, "PTT recognizer rung failed captureId=$captureId rung=$candidate: ${err.message}") + } + } + throw lastFailure ?: IllegalStateException("Speech recognizer unavailable") + } + + private fun pushToTalkRecognizerIntent(rung: PushToTalkRecognitionRung): Intent = + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_LANGUAGE, resolvedSpeechLocaleTag()) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) + putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3) + putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) + when (rung) { + is PushToTalkRecognitionRung.RawAudioSegmented -> + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + applyRawAudioSegmentedExtras(this, rung.source) + } + PushToTalkRecognitionRung.SilenceSegmented -> { + putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, 2500) + putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, 1800) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + applySilenceSegmentedExtras(this) + } + } + PushToTalkRecognitionRung.RestartingSingleSession -> { + putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, 2500) + putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, 1800) + } + } + } + + // API 33 RecognizerIntent extras live behind @RequiresApi so min-SDK lint stays meaningful; + // segmented rungs are only ever constructed on TIRAMISU+ (see pushToTalkRecognitionCandidates). + @RequiresApi(Build.VERSION_CODES.TIRAMISU) + private fun applyRawAudioSegmentedExtras( + intent: Intent, + source: PushToTalkAudioSource, + ) { + intent.putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE, source.readDescriptor) + intent.putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_CHANNEL_COUNT, 1) + intent.putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_ENCODING, AudioFormat.ENCODING_PCM_16BIT) + intent.putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_SAMPLING_RATE, pushToTalkSampleRateHz) + intent.putExtra(RecognizerIntent.EXTRA_SEGMENTED_SESSION, RecognizerIntent.EXTRA_AUDIO_SOURCE) + } + + @RequiresApi(Build.VERSION_CODES.TIRAMISU) + private fun applySilenceSegmentedExtras(intent: Intent) { + intent.putExtra( + RecognizerIntent.EXTRA_SEGMENTED_SESSION, + RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, + ) + } + + @SuppressLint("MissingPermission") + private fun openPushToTalkAudioSource(): PushToTalkAudioSource { + check(Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) + val minBufferSize = + AudioRecord.getMinBufferSize( + pushToTalkSampleRateHz, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + ) + check(minBufferSize > 0) { "AudioRecord buffer unavailable" } + + val pipe = ParcelFileDescriptor.createPipe() + var recorder: AudioRecord? = null + var writeStream: ParcelFileDescriptor.AutoCloseOutputStream? = null + try { + recorder = + AudioRecord( + MediaRecorder.AudioSource.VOICE_RECOGNITION, + pushToTalkSampleRateHz, + AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT, + minBufferSize * 2, + ) + check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord initialization failed" } + recorder.startRecording() + check(recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) { "AudioRecord did not start" } + val activeRecorder = checkNotNull(recorder) + val activeWriteStream = ParcelFileDescriptor.AutoCloseOutputStream(pipe[1]) + writeStream = activeWriteStream + val source = PushToTalkAudioSource(pipe[0], activeWriteStream, activeRecorder) + source.pumpJob = + gatewayWorkScope.launch(Dispatchers.IO) { + val buffer = ByteArray(minBufferSize.coerceAtLeast(4_096)) + try { + while (currentCoroutineContext().isActive) { + val bytesRead = activeRecorder.read(buffer, 0, buffer.size) + if (bytesRead <= 0) break + _inputLevel.value = + TalkAudioLevel.smoothed(_inputLevel.value, TalkAudioLevel.pcm16Level(buffer, bytesRead)) + activeWriteStream.write(buffer, 0, bytesRead) + } + } catch (err: IOException) { + Log.d(tag, "PTT audio pipe closed: ${err.message}") + } finally { + source.finishFromPump() + } + } + return source + } catch (err: Throwable) { + runCatching { recorder?.stop() } + runCatching { recorder?.release() } + runCatching { writeStream?.close() } + if (writeStream == null) runCatching { pipe[1].close() } + runCatching { pipe[0].close() } + throw err + } + } + + private fun schedulePushToTalkRestart( + delayMs: Long, + advanceRung: Boolean, + ) { + val captureId = activePttCaptureId ?: return + if (pttReleaseCompletion != null) return + val rung = pttRecognitionRung ?: return + val firstCandidate = + when (rung) { + is PushToTalkRecognitionRung.RawAudioSegmented -> + if (advanceRung) { + PushToTalkRecognitionCandidate.SilenceSegmented + } else { + PushToTalkRecognitionCandidate.RawAudioSegmented + } + PushToTalkRecognitionRung.SilenceSegmented -> + if (advanceRung) { + PushToTalkRecognitionCandidate.RestartingSingleSession + } else { + PushToTalkRecognitionCandidate.SilenceSegmented + } + PushToTalkRecognitionRung.RestartingSingleSession -> + PushToTalkRecognitionCandidate.RestartingSingleSession + } + commitPushToTalkLivePartial() + closePushToTalkRung() + restartJob?.cancel() + restartJob = + gatewayWorkScope.launch { + delay(delayMs) + mainHandler.post { + if (activePttCaptureId != captureId || pttReleaseCompletion != null || stopRequested) return@post + try { + startPushToTalkRecognition(captureId, firstCandidate) + } catch (err: Throwable) { + _isListening.value = false + setTalkFailure(nativeText("Talk failed: \$message", err.message ?: err::class.simpleName.orEmpty())) + } + } + } + } + + private fun closePushToTalkRung() { + (pttRecognitionRung as? PushToTalkRecognitionRung.RawAudioSegmented)?.source?.close() + pttRecognitionRung = null + } + + private fun commitPushToTalkLivePartial() { + val partial = pttLivePartial.trim() + if (partial.isNotEmpty()) { + pttFinalSegments += partial + } + pttLivePartial = "" + } + private fun startListeningInternal(markListening: Boolean) { val r = recognizer ?: return val intent = @@ -2153,6 +2465,18 @@ class TalkModeManager internal constructor( isFinal: Boolean, ) { val trimmed = text.trim() + if (activePttCaptureId != null) { + if (trimmed.isNotEmpty()) { + if (isFinal) { + pttFinalSegments += trimmed + pttLivePartial = "" + } else { + pttLivePartial = trimmed + } + lastHeardAtMs = SystemClock.elapsedRealtime() + } + return + } if (_isSpeaking.value && interruptOnSpeech) { if (shouldInterrupt(trimmed)) { stopSpeaking() @@ -2188,13 +2512,19 @@ class TalkModeManager internal constructor( private fun checkSilence(captureId: String) { if (!_isListening.value) return - val transcript = lastTranscript.trim() + val transcript = + if (activePttCaptureId != null) { + PushToTalkTranscriptMerger.merge(pttFinalSegments, pttLivePartial) + } else { + lastTranscript.trim() + } if (transcript.isEmpty()) return val lastHeard = lastHeardAtMs ?: return val elapsed = SystemClock.elapsedRealtime() - lastHeard if (elapsed < silenceWindowMs) return if (activePttCaptureId != null) { if (pttAutoStopEnabled) { + if (pttReleaseCompletion != null) return gatewayWorkScope.launch { endPushToTalk(captureId) } } return @@ -2285,14 +2615,78 @@ class TalkModeManager internal constructor( } } + private suspend fun awaitPushToTalkRelease(captureId: String) { + if (activePttCaptureId != captureId) return + restartJob?.cancel() + restartJob = null + val rung = pttRecognitionRung ?: return + // onResults, onError, and onEndOfSegmentedSession normally arrive well under a second, + // so typical release latency is unchanged. The five-second bound only caps pathological recognizers; + // leaving early truncates final words, which is worse than waiting. + pttReleaseCompletion?.let { existing -> + awaitPushToTalkReleaseCompletion(existing, pushToTalkReleaseGraceMs) + return + } + if (!_isListening.value || recognizer == null) return + + val completion = CompletableDeferred() + pttReleaseCompletion = completion + _isListening.value = false + _inputLevel.value = 0f + when (rung) { + is PushToTalkRecognitionRung.RawAudioSegmented -> { + rung.source.requestFinish() + // EXTRA_AUDIO_SOURCE is optional: a service may ignore the pipe and run its own mic, + // so closing our AudioRecord alone would leave it listening past release. stopListening + // forces its endpointer; for pipe-consuming services it is redundant after EOF. + runCatching { recognizer?.stopListening() }.onFailure { completion.complete(Unit) } + } + PushToTalkRecognitionRung.SilenceSegmented, + PushToTalkRecognitionRung.RestartingSingleSession, + -> runCatching { recognizer?.stopListening() }.onFailure { completion.complete(Unit) } + } + awaitPushToTalkReleaseCompletion(completion, pushToTalkReleaseGraceMs) + if (pttReleaseCompletion === completion) { + pttReleaseCompletion = null + } + } + + private suspend fun drainPushToTalkReleaseBeforeBegin() { + val deadline = SystemClock.elapsedRealtime() + pushToTalkReleaseDrainTimeoutMs + while (true) { + val remainingMs = deadline - SystemClock.elapsedRealtime() + val release = pttReleaseCompletion + if (release != null && remainingMs > 0) { + awaitPushToTalkReleaseCompletion(release, remainingMs) + } + if (activePttCaptureId == null && pttReleaseCompletion == null) return + if (SystemClock.elapsedRealtime() >= deadline) return + yield() + } + } + + private suspend fun awaitPushToTalkReleaseCompletion( + completion: CompletableDeferred, + timeoutMs: Long, + ) { + try { + withTimeoutOrNull(timeoutMs) { completion.await() } + } catch (err: CancellationException) { + if (completion.isCancelled && currentCoroutineContext().isActive) return + throw err + } + } + private fun clearPushToTalkRecognition(captureId: String): ClearedPushToTalkCapture? { if (activePttCaptureId != captureId) return null - val transcript = lastTranscript.trim() + val transcript = PushToTalkTranscriptMerger.merge(pttFinalSegments, pttLivePartial) val completion = pttCompletion pttTimeoutJob?.cancel() pttTimeoutJob = null pttAutoStopEnabled = false pttCompletion = null + pttReleaseCompletion?.cancel() + pttReleaseCompletion = null activePttCaptureId = null _isListening.value = false listeningMode = false @@ -2300,8 +2694,12 @@ class TalkModeManager internal constructor( recognizer?.cancel() recognizer?.destroy() recognizer = null + closePushToTalkRung() + pttFinalSegments.clear() + pttLivePartial = "" lastTranscript = "" lastHeardAtMs = null + _inputLevel.value = 0f return ClearedPushToTalkCapture(transcript = transcript, completion = completion) } @@ -2966,24 +3364,39 @@ class TalkModeManager internal constructor( } } - private val listener = + private val listener = recognitionListener(captureId = null) + + private fun recognitionListener(captureId: String?): RecognitionListener = object : RecognitionListener { override fun onReadyForSpeech(params: Bundle?) { + if (!acceptRecognitionCallback(captureId)) return // Only a live listening session may claim the status; a speech-interrupt // recognizer readying during playback must not touch Thinking state. - if (_isEnabled.value && _isListening.value) { + if (activePttCaptureId != null && _isListening.value) { + setStatus(nativeText("Listening (PTT)")) + } else if (_isEnabled.value && _isListening.value) { setStatus(nativeText("Listening")) } } - override fun onBeginningOfSpeech() {} + override fun onBeginningOfSpeech() { + if (!acceptRecognitionCallback(captureId)) return + } - override fun onRmsChanged(rmsdB: Float) {} + override fun onRmsChanged(rmsdB: Float) { + if (!acceptRecognitionCallback(captureId)) return + if (activePttCaptureId != null && pttRecognitionRung !is PushToTalkRecognitionRung.RawAudioSegmented) { + _inputLevel.value = ((rmsdB + 2f) / 12f).coerceIn(0f, 1f) + } + } override fun onBufferReceived(buffer: ByteArray?) {} override fun onEndOfSpeech() { + if (!acceptRecognitionCallback(captureId)) return clearListenWatchdog() + _inputLevel.value = 0f + if (activePttCaptureId != null) return // Don't restart while a transcript is being processed — the recognizer // competing for audio resources kills AudioTrack PCM playback. if (!finalizeInFlight) { @@ -2992,8 +3405,38 @@ class TalkModeManager internal constructor( } override fun onError(error: Int) { + if (!acceptRecognitionCallback(captureId)) return if (stopRequested) return _isListening.value = false + _inputLevel.value = 0f + if (activePttCaptureId != null) { + pttReleaseCompletion?.let { + it.complete(Unit) + return + } + if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) { + setStatus(nativeText("Microphone permission required")) + return + } + setStatus( + when (error) { + SpeechRecognizer.ERROR_AUDIO -> nativeText("Audio error") + SpeechRecognizer.ERROR_CLIENT -> nativeText("Client error") + SpeechRecognizer.ERROR_NETWORK -> nativeText("Network error") + SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> nativeText("Network timeout") + SpeechRecognizer.ERROR_NO_MATCH -> nativeText("Listening (PTT)") + SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> nativeText("Recognizer busy") + SpeechRecognizer.ERROR_SERVER -> nativeText("Server error") + SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> nativeText("Listening (PTT)") + else -> nativeText("Speech error (\$error)", error) + }, + ) + schedulePushToTalkRestart( + delayMs = 600L, + advanceRung = pttRecognitionRung !is PushToTalkRecognitionRung.RestartingSingleSession, + ) + return + } if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) { setStatus(nativeText("Microphone permission required")) return @@ -3016,21 +3459,60 @@ class TalkModeManager internal constructor( } override fun onResults(results: Bundle?) { + if (!acceptRecognitionCallback(captureId)) return val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() list.firstOrNull()?.let { handleTranscript(it, isFinal = true) } + if (activePttCaptureId != null) { + _isListening.value = false + _inputLevel.value = 0f + pttReleaseCompletion?.let { + it.complete(Unit) + return + } + schedulePushToTalkRestart( + delayMs = pushToTalkRestartDelayMs, + advanceRung = pttRecognitionRung !is PushToTalkRecognitionRung.RestartingSingleSession, + ) + return + } scheduleRestart() } override fun onPartialResults(partialResults: Bundle?) { + if (!acceptRecognitionCallback(captureId)) return val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() list.firstOrNull()?.let { handleTranscript(it, isFinal = false) } } + override fun onSegmentResults(segmentResults: Bundle) { + if (!acceptRecognitionCallback(captureId) || activePttCaptureId == null) return + val list = segmentResults.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty() + list.firstOrNull()?.let { handleTranscript(it, isFinal = true) } + } + + override fun onEndOfSegmentedSession() { + if (!acceptRecognitionCallback(captureId) || activePttCaptureId == null) return + _isListening.value = false + _inputLevel.value = 0f + pttReleaseCompletion?.let { + it.complete(Unit) + return + } + schedulePushToTalkRestart( + delayMs = 180L, + advanceRung = shouldAdvancePushToTalkRungAfterSegmentedSession(pttRecognitionRung?.candidate ?: return), + ) + } + override fun onEvent( eventType: Int, params: Bundle?, ) {} } + + // SpeechRecognizer can post callbacks after destroy. Binding each listener to its + // capture prevents a retired session from mutating normal Talk or the next PTT hold. + private fun acceptRecognitionCallback(captureId: String?): Boolean = captureId == activePttCaptureId } private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject diff --git a/apps/android/app/src/test/java/ai/openclaw/app/voice/PushToTalkRecognitionLadderTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/voice/PushToTalkRecognitionLadderTest.kt new file mode 100644 index 000000000000..80bb64025e95 --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/app/voice/PushToTalkRecognitionLadderTest.kt @@ -0,0 +1,56 @@ +package ai.openclaw.app.voice + +import org.junit.Assert.assertEquals +import org.junit.Test + +class PushToTalkRecognitionLadderTest { + @Test + fun api33StartsWithRawAudioThenFallsThroughInOrder() { + assertEquals( + listOf( + PushToTalkRecognitionCandidate.RawAudioSegmented, + PushToTalkRecognitionCandidate.SilenceSegmented, + PushToTalkRecognitionCandidate.RestartingSingleSession, + ), + pushToTalkRecognitionCandidates(supportsSegmentedRecognition = true, first = null), + ) + } + + @Test + fun olderApisUseRestartingSingleSessionOnly() { + assertEquals( + listOf(PushToTalkRecognitionCandidate.RestartingSingleSession), + pushToTalkRecognitionCandidates(supportsSegmentedRecognition = false, first = null), + ) + } + + @Test + fun degradedHoldNeverClimbsBackUpTheLadder() { + assertEquals( + listOf( + PushToTalkRecognitionCandidate.SilenceSegmented, + PushToTalkRecognitionCandidate.RestartingSingleSession, + ), + pushToTalkRecognitionCandidates( + supportsSegmentedRecognition = true, + first = PushToTalkRecognitionCandidate.SilenceSegmented, + ), + ) + } + + @Test + fun rawSegmentedSessionEndAdvancesToSilenceButSilenceSessionEndRearms() { + assertEquals( + true, + shouldAdvancePushToTalkRungAfterSegmentedSession( + PushToTalkRecognitionCandidate.RawAudioSegmented, + ), + ) + assertEquals( + false, + shouldAdvancePushToTalkRungAfterSegmentedSession( + PushToTalkRecognitionCandidate.SilenceSegmented, + ), + ) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/app/voice/PushToTalkTranscriptMergerTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/voice/PushToTalkTranscriptMergerTest.kt new file mode 100644 index 000000000000..bc3b5879ba9e --- /dev/null +++ b/apps/android/app/src/test/java/ai/openclaw/app/voice/PushToTalkTranscriptMergerTest.kt @@ -0,0 +1,90 @@ +package ai.openclaw.app.voice + +import org.junit.Assert.assertEquals +import org.junit.Test + +class PushToTalkTranscriptMergerTest { + @Test + fun joinsMultipleFinalSegmentsInOrder() { + assertEquals( + "first thought. second thought. third thought", + PushToTalkTranscriptMerger.merge( + finalSegments = listOf("first thought", "second thought", "third thought"), + livePartial = null, + ), + ) + } + + @Test + fun preservesTerminalPunctuationWithoutDoublingIt() { + assertEquals( + "ready? yes! \"done.\" next", + PushToTalkTranscriptMerger.merge( + finalSegments = listOf("ready?", "yes!", "\"done.\"", "next"), + livePartial = null, + ), + ) + } + + @Test + fun preservesLocalePunctuationWithoutInjectingAsciiSeparators() { + assertEquals( + "你好。 世界", + PushToTalkTranscriptMerger.merge( + finalSegments = listOf("你好。", "世界"), + livePartial = null, + ), + ) + assertEquals( + "هل أنت جاهز؟ نعم", + PushToTalkTranscriptMerger.merge( + finalSegments = listOf("هل أنت جاهز؟", "نعم"), + livePartial = null, + ), + ) + } + + @Test + fun omitsTrailingPartialThatDuplicatesLastFinal() { + assertEquals( + "first. SAME words", + PushToTalkTranscriptMerger.merge( + finalSegments = listOf("first", "SAME words"), + livePartial = " same WORDS ", + ), + ) + } + + @Test + fun appendsDistinctTrailingPartial() { + assertEquals( + "first final. trailing words", + PushToTalkTranscriptMerger.merge( + finalSegments = listOf("first final"), + livePartial = "trailing words", + ), + ) + } + + @Test + fun returnsPartialForPartialOnlyHold() { + assertEquals( + "unfinished thought", + PushToTalkTranscriptMerger.merge( + finalSegments = emptyList(), + livePartial = " unfinished thought ", + ), + ) + } + + @Test + fun returnsEmptyForEmptyResults() { + assertEquals( + "", + PushToTalkTranscriptMerger.merge( + finalSegments = listOf("", " "), + livePartial = " ", + ), + ) + } +} diff --git a/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt b/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt index 2a849506db03..9f1e79573051 100644 --- a/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt +++ b/apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt @@ -11,8 +11,11 @@ import ai.openclaw.app.i18n.verbatimText import android.Manifest import android.content.ComponentName import android.content.IntentFilter +import android.os.Bundle import android.os.SystemClock +import android.speech.RecognitionListener import android.speech.RecognitionService +import android.speech.SpeechRecognizer import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope @@ -26,6 +29,7 @@ import kotlinx.coroutines.async import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.advanceTimeBy import kotlinx.coroutines.test.advanceUntilIdle import kotlinx.coroutines.test.currentTime import kotlinx.coroutines.test.resetMain @@ -279,6 +283,147 @@ class TalkModeManagerTest { } } + @Test + fun segmentDuringPushToTalkReleaseWaitsForEndOfSegmentedSession() { + val manager = createManager() + val releaseCompletion = CompletableDeferred() + setPrivateField(manager, "activePttCaptureId", "capture-1") + setPrivateField(manager, "pttReleaseCompletion", releaseCompletion) + val listener = recognitionListener(manager, "capture-1") + val segment = + Bundle().apply { + putStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION, arrayListOf("first segment")) + } + + listener.onSegmentResults(segment) + + assertFalse(releaseCompletion.isCompleted) + assertEquals(listOf("first segment"), readPrivateField(manager, "pttFinalSegments")) + + listener.onEndOfSegmentedSession() + + assertTrue(releaseCompletion.isCompleted) + } + + @Test + @OptIn(ExperimentalCoroutinesApi::class) + fun releaseKeepsWaitingPastOldGraceForLateTerminalSegment() = + runTest { + val manager = createManager(isConnected = { false }) + val releaseCompletion = CompletableDeferred() + setPrivateField(manager, "activePttCaptureId", "capture-1") + setPrivateField(manager, "pttReleaseCompletion", releaseCompletion) + setPrivateField(manager, "pttRecognitionRung", silenceSegmentedRung()) + @Suppress("UNCHECKED_CAST") + (readPrivateField(manager, "pttFinalSegments") as MutableList) += "early segment" + val listener = recognitionListener(manager, "capture-1") + Dispatchers.setMain(StandardTestDispatcher(testScheduler)) + try { + val ending = async { manager.endPushToTalk("capture-1") } + runCurrent() + + advanceTimeBy(1_200) + listener.onSegmentResults( + Bundle().apply { + putStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION, arrayListOf("late segment")) + }, + ) + assertFalse(ending.isCompleted) + + listener.onEndOfSegmentedSession() + advanceUntilIdle() + + assertEquals("early segment. late segment", ending.await().transcript) + } finally { + Dispatchers.resetMain() + } + } + + @Test + @OptIn(ExperimentalCoroutinesApi::class) + fun cancelledEndPushToTalkClearsPendingReleaseBeforeNextBegin() = + runTest { + val app = RuntimeEnvironment.getApplication() + shadowOf(app).grantPermissions(Manifest.permission.RECORD_AUDIO) + val packageManager = shadowOf(app.packageManager) + val speechService = ComponentName(app, "TestSpeechRecognitionService") + packageManager.addServiceIfNotPresent(speechService) + packageManager.addIntentFilterForService(speechService, IntentFilter(RecognitionService.SERVICE_INTERFACE)) + val manager = createManager() + setPrivateField(manager, "activePttCaptureId", "capture-a") + setPrivateField(manager, "pttReleaseCompletion", CompletableDeferred()) + setPrivateField(manager, "pttRecognitionRung", silenceSegmentedRung()) + @Suppress("UNCHECKED_CAST") + (readPrivateField(manager, "pttFinalSegments") as MutableList) += "capture a" + Dispatchers.setMain(StandardTestDispatcher(testScheduler)) + try { + val ending = async { manager.endPushToTalk("capture-a") } + runCurrent() + ending.cancel() + runCurrent() + ending.join() + + assertTrue(ending.isCancelled) + assertNull(readPrivateField(manager, "activePttCaptureId")) + assertNull(readPrivateField(manager, "pttReleaseCompletion")) + assertEquals(emptyList(), readPrivateField(manager, "pttFinalSegments")) + + val started = manager.beginPushToTalk(allowNewCapture = true) + + assertEquals(started.captureId, readPrivateField(manager, "activePttCaptureId")) + assertEquals(emptyList(), readPrivateField(manager, "pttFinalSegments")) + } finally { + manager.stopAllCapture() + Dispatchers.resetMain() + } + } + + @Test + @OptIn(ExperimentalCoroutinesApi::class) + fun replacementBeginDrainsPendingReleaseBeforeStartingNewCapture() = + runTest { + val app = RuntimeEnvironment.getApplication() + shadowOf(app).grantPermissions(Manifest.permission.RECORD_AUDIO) + val packageManager = shadowOf(app.packageManager) + val speechService = ComponentName(app, "TestSpeechRecognitionService") + packageManager.addServiceIfNotPresent(speechService) + packageManager.addIntentFilterForService(speechService, IntentFilter(RecognitionService.SERVICE_INTERFACE)) + var connectionChecks = 0 + val manager = + createManager( + isConnected = { + connectionChecks += 1 + connectionChecks != 2 + }, + ) + val releaseCompletion = CompletableDeferred() + setPrivateField(manager, "activePttCaptureId", "capture-a") + setPrivateField(manager, "pttReleaseCompletion", releaseCompletion) + setPrivateField(manager, "pttRecognitionRung", silenceSegmentedRung()) + @Suppress("UNCHECKED_CAST") + (readPrivateField(manager, "pttFinalSegments") as MutableList) += "first segment" + Dispatchers.setMain(StandardTestDispatcher(testScheduler)) + try { + val ending = async { manager.endPushToTalk("capture-a") } + runCurrent() + val starting = async { manager.beginPushToTalk(allowNewCapture = true) } + runCurrent() + + releaseCompletion.complete(Unit) + advanceUntilIdle() + + val ended = ending.await() + val started = starting.await() + assertEquals("offline", ended.status) + assertEquals("first segment", ended.transcript) + assertEquals(started.captureId, readPrivateField(manager, "activePttCaptureId")) + assertEquals(emptyList(), readPrivateField(manager, "pttFinalSegments")) + } finally { + manager.stopAllCapture() + Dispatchers.resetMain() + } + } + @Test fun duplicateFinalForPendingTalkRunDoesNotStartAllResponseTts() { val manager = createManager() @@ -1125,6 +1270,20 @@ class TalkModeManagerTest { return method.invoke(manager, length) as Boolean } + private fun recognitionListener( + manager: TalkModeManager, + captureId: String, + ): RecognitionListener { + val method = manager.javaClass.getDeclaredMethod("recognitionListener", String::class.java) + method.isAccessible = true + return method.invoke(manager, captureId) as RecognitionListener + } + + private fun silenceSegmentedRung(): Any { + val clazz = Class.forName("ai.openclaw.app.voice.PushToTalkRecognitionRung\$SilenceSegmented") + return requireNotNull(clazz.getField("INSTANCE").get(null)) + } + private fun chatFinalPayload( runId: String, text: String,