mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix(android): stop push-to-talk from dropping speech after mid-hold pauses (#110995)
* fix(android): stop push-to-talk from dropping speech after mid-hold pauses The SpeechRecognizer PTT path ran one session with a 2.5s silence window, kept only the last final, and never restarted during a hold - speech after a mid-hold pause was silently dropped. Capture now uses an ordered rung ladder: API 33+ app-owned AudioRecord piped via EXTRA_AUDIO_SOURCE with EXTRA_SEGMENTED_SESSION, then silence-keyed segmented sessions, then a restart-during-hold single-session fallback for older devices; failed holds only descend. Final segments accumulate and merge locale-aware (no ASCII separators after CJK/Arabic punctuation), with a trailing live partial deduped against the last final. Release waits one bounded grace for the terminal recognizer callback so late finals are not truncated; rapid re-press drains the prior release instead of cancelling it; cancellation mid-release still tears capture down, resumes realtime talk, and resolves the stop payload. Raw-rung release also stops the recognizer since EXTRA_AUDIO_SOURCE is optional and a service may run its own microphone. Input level meters from our PCM stream in the raw rung where onRmsChanged never fires. * fix(android): API-33 recognizer extras behind RequiresApi and inventory refresh Min-SDK lint (InlinedApi) flagged the API 33 RecognizerIntent extras; they now live in RequiresApi(TIRAMISU) helpers behind explicit SDK gates, and the native i18n inventory is refreshed for the moved lines. * chore(android): rebuild native locale artifacts for inventory order
This commit is contained in:
committed by
GitHub
parent
ea54060223
commit
c84a59284b
@@ -13043,7 +13043,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 532,
|
||||
"line": 602,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Speech recognizer unavailable",
|
||||
"surface": "android",
|
||||
@@ -13051,15 +13051,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 596,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Listening (PTT)",
|
||||
"surface": "android",
|
||||
"id": "native.android.0faf9df3e5f708df"
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 693,
|
||||
"line": 800,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Ready",
|
||||
"surface": "android",
|
||||
@@ -13067,7 +13059,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 911,
|
||||
"line": 1018,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Start failed: $message",
|
||||
"surface": "android",
|
||||
@@ -13075,7 +13067,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 1001,
|
||||
"line": 1114,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Connecting…",
|
||||
"surface": "android",
|
||||
@@ -13083,7 +13075,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 1070,
|
||||
"line": 1183,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Off",
|
||||
"surface": "android",
|
||||
@@ -13091,7 +13083,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 1073,
|
||||
"line": 1186,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Talk failed: Realtime provider closed unexpectedly.",
|
||||
"surface": "android",
|
||||
@@ -13099,7 +13091,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 1078,
|
||||
"line": 1191,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Talk failed: Realtime provider closed: $reason",
|
||||
"surface": "android",
|
||||
@@ -13107,7 +13099,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2216,
|
||||
"line": 2546,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Thinking…",
|
||||
"surface": "android",
|
||||
@@ -13115,7 +13107,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2233,
|
||||
"line": 2563,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Gateway not connected",
|
||||
"surface": "android",
|
||||
@@ -13123,7 +13115,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2246,
|
||||
"line": 2576,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Aborted",
|
||||
"surface": "android",
|
||||
@@ -13131,7 +13123,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2246,
|
||||
"line": 2576,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Chat error",
|
||||
"surface": "android",
|
||||
@@ -13139,7 +13131,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2263,
|
||||
"line": 2593,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "No reply",
|
||||
"surface": "android",
|
||||
@@ -13147,7 +13139,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2279,
|
||||
"line": 2609,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Talk failed: $message",
|
||||
"surface": "android",
|
||||
@@ -13155,7 +13147,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2504,
|
||||
"line": 2902,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Generating voice…",
|
||||
"surface": "android",
|
||||
@@ -13163,7 +13155,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2532,
|
||||
"line": 2930,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Speak failed: $message",
|
||||
"surface": "android",
|
||||
@@ -13171,7 +13163,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2681,
|
||||
"line": 3079,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Speaking…",
|
||||
"surface": "android",
|
||||
@@ -13179,7 +13171,15 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 2998,
|
||||
"line": 3430,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Listening (PTT)",
|
||||
"surface": "android",
|
||||
"id": "native.android.0faf9df3e5f708df"
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3441,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Microphone permission required",
|
||||
"surface": "android",
|
||||
@@ -13187,7 +13187,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3004,
|
||||
"line": 3447,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Audio error",
|
||||
"surface": "android",
|
||||
@@ -13195,7 +13195,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3005,
|
||||
"line": 3448,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Client error",
|
||||
"surface": "android",
|
||||
@@ -13203,7 +13203,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3006,
|
||||
"line": 3449,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Network error",
|
||||
"surface": "android",
|
||||
@@ -13211,7 +13211,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3007,
|
||||
"line": 3450,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Network timeout",
|
||||
"surface": "android",
|
||||
@@ -13219,7 +13219,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3009,
|
||||
"line": 3452,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Recognizer busy",
|
||||
"surface": "android",
|
||||
@@ -13227,7 +13227,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3010,
|
||||
"line": 3453,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Server error",
|
||||
"surface": "android",
|
||||
@@ -13235,7 +13235,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3011,
|
||||
"line": 3454,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Listening",
|
||||
"surface": "android",
|
||||
@@ -13243,7 +13243,7 @@
|
||||
},
|
||||
{
|
||||
"kind": "ui-call",
|
||||
"line": 3012,
|
||||
"line": 3455,
|
||||
"path": "apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt",
|
||||
"source": "Speech error ($error)",
|
||||
"surface": "android",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "أداة التعرّف على الكلام غير متاحة"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "جارٍ الاستماع (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "جارٍ التحدث…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "جارٍ الاستماع (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Spracherkennung nicht verfügbar"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Hört zu (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Spricht…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Hört zu (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "El reconocimiento de voz no está disponible"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Escuchando (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Hablando…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Escuchando (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "تشخیصدهنده گفتار در دسترس نیست"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "در حال شنیدن (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "در حال صحبت…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "در حال شنیدن (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Reconnaissance vocale indisponible"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Écoute (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Lecture en cours…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Écoute (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "वाक् पहचानकर्ता उपलब्ध नहीं है"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "सुन रहा है (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "बोल रहा है…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "सुन रहा है (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Pengenal ucapan tidak tersedia"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Mendengarkan (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Berbicara…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Mendengarkan (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Riconoscimento vocale non disponibile"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "In ascolto (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Riproduzione vocale…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "In ascolto (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "音声認識を利用できません"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "聞き取り中(PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "読み上げ中…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "聞き取り中(PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "음성 인식기를 사용할 수 없습니다"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "듣는 중(PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "말하는 중…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "듣는 중(PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Spraakherkenning niet beschikbaar"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Luisteren (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Spreken…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Luisteren (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Rozpoznawanie mowy jest niedostępne"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Nasłuchiwanie (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Mówienie…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Nasłuchiwanie (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Reconhecimento de fala indisponível"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Ouvindo (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Falando…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Ouvindo (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Распознавание речи недоступно"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Прослушивание (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Воспроизведение речи…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Прослушивание (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Taligenkänning är inte tillgänglig"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Lyssnar (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Talar…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Lyssnar (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "ไม่สามารถใช้ตัวจดจำเสียงพูดได้"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "กำลังฟัง (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "กำลังพูด…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "กำลังฟัง (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Konuşma tanıyıcı kullanılamıyor"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Dinleniyor (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Konuşuyor…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Dinleniyor (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Розпізнавання мовлення недоступне"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Прослуховування (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Озвучення…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Прослуховування (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "Trình nhận dạng giọng nói không khả dụng"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Đang nghe (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "Đang nói…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "Đang nghe (PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "语音识别器不可用"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "正在聆听(按住说话)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "正在朗读…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "正在聆听(按住说话)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -8158,11 +8158,6 @@
|
||||
"source": "Speech recognizer unavailable",
|
||||
"translated": "語音辨識器無法使用"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "正在聆聽(PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.47a59f8798682ea0",
|
||||
"source": "Ready",
|
||||
@@ -8238,6 +8233,11 @@
|
||||
"source": "Speaking…",
|
||||
"translated": "正在朗讀…"
|
||||
},
|
||||
{
|
||||
"id": "native.android.0faf9df3e5f708df",
|
||||
"source": "Listening (PTT)",
|
||||
"translated": "正在聆聽(PTT)"
|
||||
},
|
||||
{
|
||||
"id": "native.android.d1a7db2c16fd6d09",
|
||||
"source": "Microphone permission required",
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package ai.openclaw.app.voice
|
||||
|
||||
internal enum class PushToTalkRecognitionCandidate {
|
||||
RawAudioSegmented,
|
||||
SilenceSegmented,
|
||||
RestartingSingleSession,
|
||||
}
|
||||
|
||||
internal fun pushToTalkRecognitionCandidates(
|
||||
supportsSegmentedRecognition: Boolean,
|
||||
first: PushToTalkRecognitionCandidate?,
|
||||
): List<PushToTalkRecognitionCandidate> {
|
||||
val available =
|
||||
if (supportsSegmentedRecognition) {
|
||||
PushToTalkRecognitionCandidate.entries
|
||||
} else {
|
||||
listOf(PushToTalkRecognitionCandidate.RestartingSingleSession)
|
||||
}
|
||||
if (first == null) return available
|
||||
return available.dropWhile { it != first }.ifEmpty { listOf(PushToTalkRecognitionCandidate.RestartingSingleSession) }
|
||||
}
|
||||
|
||||
internal fun shouldAdvancePushToTalkRungAfterSegmentedSession(
|
||||
candidate: PushToTalkRecognitionCandidate,
|
||||
): Boolean = candidate == PushToTalkRecognitionCandidate.RawAudioSegmented
|
||||
@@ -0,0 +1,39 @@
|
||||
package ai.openclaw.app.voice
|
||||
|
||||
import java.util.Locale
|
||||
|
||||
internal object PushToTalkTranscriptMerger {
|
||||
private val trailingClosers = setOf('"', '\'', '’', '”', ')', ']', '}')
|
||||
|
||||
fun merge(
|
||||
finalSegments: List<String>,
|
||||
livePartial: String?,
|
||||
): String {
|
||||
val segments = finalSegments.map { it.trim() }.filter { it.isNotEmpty() }.toMutableList()
|
||||
val partial = livePartial?.trim().orEmpty()
|
||||
val lastFinal = segments.lastOrNull()
|
||||
if (partial.isNotEmpty() && (lastFinal == null || normalize(partial) != normalize(lastFinal))) {
|
||||
segments += partial
|
||||
}
|
||||
|
||||
return buildString {
|
||||
var previousEndsSentence = false
|
||||
segments.forEachIndexed { index, segment ->
|
||||
if (index > 0) {
|
||||
append(if (previousEndsSentence) " " else ". ")
|
||||
}
|
||||
append(segment)
|
||||
// Recognizers emit locale punctuation (。?؟…); injecting ASCII ". " after those corrupts
|
||||
// the transcript. Separate with ". " only when a segment ends in a letter or digit.
|
||||
val finalCharacter = segment.trimEnd { it in trailingClosers }.lastOrNull()
|
||||
previousEndsSentence = finalCharacter != null && !finalCharacter.isLetterOrDigit()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun normalize(value: String): String =
|
||||
value
|
||||
.trim()
|
||||
.lowercase(Locale.ROOT)
|
||||
.replace(Regex("""\s+"""), " ")
|
||||
}
|
||||
@@ -17,10 +17,14 @@ import android.media.AudioAttributes
|
||||
import android.media.AudioFocusRequest
|
||||
import android.media.AudioFormat
|
||||
import android.media.AudioManager
|
||||
import android.media.AudioRecord
|
||||
import android.media.AudioTrack
|
||||
import android.media.MediaRecorder
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.os.ParcelFileDescriptor
|
||||
import android.os.SystemClock
|
||||
import android.speech.RecognitionListener
|
||||
import android.speech.RecognizerIntent
|
||||
@@ -29,6 +33,7 @@ import android.speech.tts.TextToSpeech
|
||||
import android.speech.tts.UtteranceProgressListener
|
||||
import android.util.Base64
|
||||
import android.util.Log
|
||||
import androidx.annotation.RequiresApi
|
||||
import androidx.core.content.ContextCompat
|
||||
import kotlinx.coroutines.CancellationException
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
@@ -53,15 +58,19 @@ import kotlinx.coroutines.sync.Mutex
|
||||
import kotlinx.coroutines.sync.withLock
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.coroutines.withTimeout
|
||||
import kotlinx.coroutines.withTimeoutOrNull
|
||||
import kotlinx.coroutines.yield
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import java.io.IOException
|
||||
import java.util.LinkedHashMap
|
||||
import java.util.Locale
|
||||
import java.util.UUID
|
||||
import java.util.concurrent.atomic.AtomicBoolean
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import kotlin.coroutines.coroutineContext
|
||||
|
||||
@@ -183,6 +192,57 @@ private data class PendingRealtimePlaybackMark(
|
||||
var targetFrame: Long? = null,
|
||||
)
|
||||
|
||||
private class PushToTalkAudioSource(
|
||||
val readDescriptor: ParcelFileDescriptor,
|
||||
private val writeStream: ParcelFileDescriptor.AutoCloseOutputStream,
|
||||
private val audioRecord: AudioRecord,
|
||||
) {
|
||||
private val finishRequested = AtomicBoolean(false)
|
||||
private val inputFinished = AtomicBoolean(false)
|
||||
private val descriptorClosed = AtomicBoolean(false)
|
||||
var pumpJob: Job? = null
|
||||
|
||||
fun requestFinish() {
|
||||
if (!finishRequested.compareAndSet(false, true)) return
|
||||
runCatching { audioRecord.stop() }
|
||||
}
|
||||
|
||||
fun finishFromPump() {
|
||||
if (!inputFinished.compareAndSet(false, true)) return
|
||||
runCatching { audioRecord.stop() }
|
||||
runCatching { audioRecord.release() }
|
||||
runCatching { writeStream.close() }
|
||||
}
|
||||
|
||||
fun close() {
|
||||
requestFinish()
|
||||
pumpJob?.cancel()
|
||||
finishFromPump()
|
||||
pumpJob = null
|
||||
if (descriptorClosed.compareAndSet(false, true)) {
|
||||
runCatching { readDescriptor.close() }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed interface PushToTalkRecognitionRung {
|
||||
val candidate: PushToTalkRecognitionCandidate
|
||||
|
||||
data class RawAudioSegmented(
|
||||
val source: PushToTalkAudioSource,
|
||||
) : PushToTalkRecognitionRung {
|
||||
override val candidate = PushToTalkRecognitionCandidate.RawAudioSegmented
|
||||
}
|
||||
|
||||
data object SilenceSegmented : PushToTalkRecognitionRung {
|
||||
override val candidate = PushToTalkRecognitionCandidate.SilenceSegmented
|
||||
}
|
||||
|
||||
data object RestartingSingleSession : PushToTalkRecognitionRung {
|
||||
override val candidate = PushToTalkRecognitionCandidate.RestartingSingleSession
|
||||
}
|
||||
}
|
||||
|
||||
class TalkModeManager internal constructor(
|
||||
private val context: Context,
|
||||
private val scope: CoroutineScope,
|
||||
@@ -209,6 +269,10 @@ class TalkModeManager internal constructor(
|
||||
private const val maxConversationEntries = 40
|
||||
private const val realtimePlaybackBufferMs = 240
|
||||
private const val realtimeUserFinalRewriteGraceMs = 1_500L
|
||||
private const val pushToTalkSampleRateHz = 16_000
|
||||
private const val pushToTalkReleaseGraceMs = 5_000L
|
||||
private const val pushToTalkReleaseDrainTimeoutMs = 6_000L
|
||||
private const val pushToTalkRestartDelayMs = 200L
|
||||
}
|
||||
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
@@ -288,6 +352,10 @@ class TalkModeManager internal constructor(
|
||||
private var pttAutoStopEnabled = false
|
||||
private var pttTimeoutJob: Job? = null
|
||||
private var pttCompletion: CompletableDeferred<TalkPttStopPayload>? = null
|
||||
private var pttRecognitionRung: PushToTalkRecognitionRung? = null
|
||||
private var pttReleaseCompletion: CompletableDeferred<Unit>? = null
|
||||
private val pttFinalSegments = mutableListOf<String>()
|
||||
private var pttLivePartial = ""
|
||||
|
||||
private var silenceJob: Job? = null
|
||||
private var silenceWindowMs = TalkDefaults.defaultSilenceTimeoutMs
|
||||
@@ -511,7 +579,9 @@ class TalkModeManager internal constructor(
|
||||
}
|
||||
// PTT begin is idempotent so gateway retries don't start multiple recognizers.
|
||||
activePttCaptureId?.let {
|
||||
return PushToTalkStartResult.Existing(TalkPttStartPayload(captureId = it))
|
||||
if (pttReleaseCompletion == null) {
|
||||
return PushToTalkStartResult.Existing(TalkPttStartPayload(captureId = it))
|
||||
}
|
||||
}
|
||||
finishingPttCaptureId?.let {
|
||||
throw IllegalStateException("PTT_BUSY: previous push-to-talk turn is still finishing")
|
||||
@@ -537,8 +607,14 @@ class TalkModeManager internal constructor(
|
||||
val captureGeneration = startGeneration.get()
|
||||
return try {
|
||||
withContext(Dispatchers.Main) {
|
||||
val hasPendingRelease = pttReleaseCompletion != null
|
||||
if (hasPendingRelease) {
|
||||
drainPushToTalkReleaseBeforeBegin()
|
||||
}
|
||||
activePttCaptureId?.let {
|
||||
return@withContext PushToTalkStartResult.Existing(TalkPttStartPayload(captureId = it))
|
||||
if (!hasPendingRelease) {
|
||||
return@withContext PushToTalkStartResult.Existing(TalkPttStartPayload(captureId = it))
|
||||
}
|
||||
}
|
||||
finishingPttCaptureId?.let {
|
||||
throw IllegalStateException("PTT_BUSY: previous push-to-talk turn is still finishing")
|
||||
@@ -559,6 +635,10 @@ class TalkModeManager internal constructor(
|
||||
recognizer?.cancel()
|
||||
recognizer?.destroy()
|
||||
recognizer = null
|
||||
closePushToTalkRung()
|
||||
pttReleaseCompletion = null
|
||||
pttFinalSegments.clear()
|
||||
pttLivePartial = ""
|
||||
lastTranscript = ""
|
||||
lastHeardAtMs = null
|
||||
activePttCaptureId = captureId
|
||||
@@ -577,9 +657,13 @@ class TalkModeManager internal constructor(
|
||||
) {
|
||||
throw IllegalStateException("NODE_BACKGROUND_UNAVAILABLE: command requires foreground")
|
||||
}
|
||||
recognizer = SpeechRecognizer.createSpeechRecognizer(context).also { it.setRecognitionListener(listener) }
|
||||
startListeningInternal(markListening = true)
|
||||
recognizer =
|
||||
SpeechRecognizer.createSpeechRecognizer(context).also {
|
||||
it.setRecognitionListener(recognitionListener(captureId))
|
||||
}
|
||||
startPushToTalkRecognition(captureId)
|
||||
} catch (err: Throwable) {
|
||||
closePushToTalkRung()
|
||||
runCatching { recognizer?.cancel() }
|
||||
runCatching { recognizer?.destroy() }
|
||||
recognizer = null
|
||||
@@ -624,59 +708,82 @@ class TalkModeManager internal constructor(
|
||||
}
|
||||
|
||||
internal suspend fun endPushToTalk(captureId: String): TalkPttStopPayload =
|
||||
withContext(Dispatchers.Main) {
|
||||
val cleared =
|
||||
clearPushToTalkRecognition(captureId)
|
||||
?: return@withContext TalkPttStopPayload(captureId = captureId, transcript = null, status = "idle")
|
||||
val transcript = cleared.transcript
|
||||
try {
|
||||
withContext(Dispatchers.Main) {
|
||||
awaitPushToTalkRelease(captureId)
|
||||
val cleared =
|
||||
clearPushToTalkRecognition(captureId)
|
||||
?: return@withContext TalkPttStopPayload(captureId = captureId, transcript = null, status = "idle")
|
||||
val transcript = cleared.transcript
|
||||
|
||||
if (transcript.isEmpty()) {
|
||||
setStatus(if (_isEnabled.value) nativeText("Listening") else nativeText("Ready"))
|
||||
resumeRealtimeCaptureAfterPushToTalk(captureId)
|
||||
return@withContext finishPushToTalk(
|
||||
TalkPttStopPayload(captureId = captureId, transcript = null, status = "empty"),
|
||||
cleared.completion,
|
||||
)
|
||||
}
|
||||
if (transcript.isEmpty()) {
|
||||
setStatus(if (_isEnabled.value) nativeText("Listening") else nativeText("Ready"))
|
||||
resumeRealtimeCaptureAfterPushToTalk(captureId)
|
||||
return@withContext finishPushToTalk(
|
||||
TalkPttStopPayload(captureId = captureId, transcript = null, status = "empty"),
|
||||
cleared.completion,
|
||||
)
|
||||
}
|
||||
|
||||
if (!isConnected()) {
|
||||
setStatus(nativeText("Gateway not connected"))
|
||||
resumeRealtimeCaptureAfterPushToTalk(captureId)
|
||||
return@withContext finishPushToTalk(
|
||||
TalkPttStopPayload(captureId = captureId, transcript = transcript, status = "offline"),
|
||||
cleared.completion,
|
||||
)
|
||||
}
|
||||
if (!isConnected()) {
|
||||
setStatus(nativeText("Gateway not connected"))
|
||||
resumeRealtimeCaptureAfterPushToTalk(captureId)
|
||||
return@withContext finishPushToTalk(
|
||||
TalkPttStopPayload(captureId = captureId, transcript = transcript, status = "offline"),
|
||||
cleared.completion,
|
||||
)
|
||||
}
|
||||
|
||||
setStatus(nativeText("Thinking…"), awaitingAgent = true)
|
||||
lateinit var finishingJob: Job
|
||||
finishingJob =
|
||||
// Gateway-scoped so a switch drops the stale finalize; the NonCancellable
|
||||
// finally still resumes capture when the scope cancels this job.
|
||||
gatewayWorkScope.launch(start = CoroutineStart.LAZY) {
|
||||
try {
|
||||
finalizeTranscript(transcript)
|
||||
} finally {
|
||||
withContext(NonCancellable + Dispatchers.Main) {
|
||||
resumeRealtimeCaptureAfterPushToTalk(captureId)
|
||||
clearFinishingPushToTalk(captureId, finishingJob)
|
||||
setStatus(nativeText("Thinking…"), awaitingAgent = true)
|
||||
lateinit var finishingJob: Job
|
||||
finishingJob =
|
||||
// Gateway-scoped so a switch drops the stale finalize; the NonCancellable
|
||||
// finally still resumes capture when the scope cancels this job.
|
||||
gatewayWorkScope.launch(start = CoroutineStart.LAZY) {
|
||||
try {
|
||||
finalizeTranscript(transcript)
|
||||
} finally {
|
||||
withContext(NonCancellable + Dispatchers.Main) {
|
||||
resumeRealtimeCaptureAfterPushToTalk(captureId)
|
||||
clearFinishingPushToTalk(captureId, finishingJob)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Cancellation can win before a lazy coroutine enters its body, in which
|
||||
// case its finally block never runs. Completion still releases ownership.
|
||||
finishingJob.invokeOnCompletion { clearFinishingPushToTalk(captureId, finishingJob) }
|
||||
// Publish the job before it can run so stop() cannot clear ownership while
|
||||
// an untracked finalizer still uses shared chat and playback state.
|
||||
synchronized(finishingPttLock) {
|
||||
finishingPttCaptureId = captureId
|
||||
finishingPttJob = finishingJob
|
||||
finishingJob.start()
|
||||
}
|
||||
// Cancellation can win before a lazy coroutine enters its body, in which
|
||||
// case its finally block never runs. Completion still releases ownership.
|
||||
finishingJob.invokeOnCompletion { clearFinishingPushToTalk(captureId, finishingJob) }
|
||||
// Publish the job before it can run so stop() cannot clear ownership while
|
||||
// an untracked finalizer still uses shared chat and playback state.
|
||||
synchronized(finishingPttLock) {
|
||||
finishingPttCaptureId = captureId
|
||||
finishingPttJob = finishingJob
|
||||
finishingJob.start()
|
||||
finishPushToTalk(
|
||||
TalkPttStopPayload(captureId = captureId, transcript = transcript, status = "queued"),
|
||||
cleared.completion,
|
||||
)
|
||||
}
|
||||
finishPushToTalk(
|
||||
TalkPttStopPayload(captureId = captureId, transcript = transcript, status = "queued"),
|
||||
cleared.completion,
|
||||
)
|
||||
} catch (err: CancellationException) {
|
||||
// Mirror the normal termination tail: resume realtime capture, restore status, and
|
||||
// resolve the PTT completion so a cancelled end (gateway drop) cannot leave Talk
|
||||
// paused or an awaiter stuck behind the release wait.
|
||||
withContext(NonCancellable + Dispatchers.Main) {
|
||||
val cleared = clearPushToTalkRecognition(captureId)
|
||||
if (cleared != null) {
|
||||
setStatus(if (_isEnabled.value) nativeText("Listening") else nativeText("Ready"))
|
||||
resumeRealtimeCaptureAfterPushToTalk(captureId)
|
||||
finishPushToTalk(
|
||||
TalkPttStopPayload(
|
||||
captureId = captureId,
|
||||
transcript = cleared.transcript.ifEmpty { null },
|
||||
status = "cancelled",
|
||||
),
|
||||
cleared.completion,
|
||||
)
|
||||
}
|
||||
}
|
||||
throw err
|
||||
}
|
||||
|
||||
/** Cancels push-to-talk capture without sending the current transcript. */
|
||||
@@ -934,9 +1041,15 @@ class TalkModeManager internal constructor(
|
||||
restartJob = null
|
||||
silenceJob?.cancel()
|
||||
silenceJob = null
|
||||
closePushToTalkRung()
|
||||
pttReleaseCompletion?.cancel()
|
||||
pttReleaseCompletion = null
|
||||
pttFinalSegments.clear()
|
||||
pttLivePartial = ""
|
||||
lastTranscript = ""
|
||||
lastHeardAtMs = null
|
||||
_isListening.value = false
|
||||
_inputLevel.value = 0f
|
||||
setStatus(nativeText("Off"), state = TalkStatusState.Off)
|
||||
stopRealtimeRelay()
|
||||
stopSpeaking()
|
||||
@@ -2105,6 +2218,205 @@ class TalkModeManager internal constructor(
|
||||
private val transcriptSpaceAfterPunctuation =
|
||||
setOf('.', '!', '?', ',', ':', ';', ')', ']', '}', '"', '\'', '’', '”')
|
||||
|
||||
// API 33 adds segmented callbacks and caller-owned audio. Keep this ordered ladder
|
||||
// in one place: removing the restart rung makes older devices drop speech after a pause.
|
||||
private fun pushToTalkCandidates(first: PushToTalkRecognitionCandidate?): List<PushToTalkRecognitionCandidate> =
|
||||
pushToTalkRecognitionCandidates(
|
||||
supportsSegmentedRecognition = Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU,
|
||||
first = first,
|
||||
)
|
||||
|
||||
private fun startPushToTalkRecognition(
|
||||
captureId: String,
|
||||
firstCandidate: PushToTalkRecognitionCandidate? = null,
|
||||
) {
|
||||
val recognizerInstance = recognizer ?: error("Speech recognizer unavailable")
|
||||
var lastFailure: Throwable? = null
|
||||
for (candidate in pushToTalkCandidates(firstCandidate)) {
|
||||
try {
|
||||
val rung =
|
||||
when (candidate) {
|
||||
PushToTalkRecognitionCandidate.RawAudioSegmented ->
|
||||
PushToTalkRecognitionRung.RawAudioSegmented(openPushToTalkAudioSource())
|
||||
PushToTalkRecognitionCandidate.SilenceSegmented -> PushToTalkRecognitionRung.SilenceSegmented
|
||||
PushToTalkRecognitionCandidate.RestartingSingleSession ->
|
||||
PushToTalkRecognitionRung.RestartingSingleSession
|
||||
}
|
||||
pttRecognitionRung = rung
|
||||
recognizerInstance.startListening(pushToTalkRecognizerIntent(rung))
|
||||
_isListening.value = true
|
||||
setStatus(nativeText("Listening (PTT)"))
|
||||
return
|
||||
} catch (err: Throwable) {
|
||||
lastFailure = err
|
||||
closePushToTalkRung()
|
||||
Log.w(tag, "PTT recognizer rung failed captureId=$captureId rung=$candidate: ${err.message}")
|
||||
}
|
||||
}
|
||||
throw lastFailure ?: IllegalStateException("Speech recognizer unavailable")
|
||||
}
|
||||
|
||||
private fun pushToTalkRecognizerIntent(rung: PushToTalkRecognitionRung): Intent =
|
||||
Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE, resolvedSpeechLocaleTag())
|
||||
putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
|
||||
putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 3)
|
||||
putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName)
|
||||
when (rung) {
|
||||
is PushToTalkRecognitionRung.RawAudioSegmented ->
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
||||
applyRawAudioSegmentedExtras(this, rung.source)
|
||||
}
|
||||
PushToTalkRecognitionRung.SilenceSegmented -> {
|
||||
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, 2500)
|
||||
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, 1800)
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
||||
applySilenceSegmentedExtras(this)
|
||||
}
|
||||
}
|
||||
PushToTalkRecognitionRung.RestartingSingleSession -> {
|
||||
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, 2500)
|
||||
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, 1800)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// API 33 RecognizerIntent extras live behind @RequiresApi so min-SDK lint stays meaningful;
|
||||
// segmented rungs are only ever constructed on TIRAMISU+ (see pushToTalkRecognitionCandidates).
|
||||
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
||||
private fun applyRawAudioSegmentedExtras(
|
||||
intent: Intent,
|
||||
source: PushToTalkAudioSource,
|
||||
) {
|
||||
intent.putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE, source.readDescriptor)
|
||||
intent.putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_CHANNEL_COUNT, 1)
|
||||
intent.putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_ENCODING, AudioFormat.ENCODING_PCM_16BIT)
|
||||
intent.putExtra(RecognizerIntent.EXTRA_AUDIO_SOURCE_SAMPLING_RATE, pushToTalkSampleRateHz)
|
||||
intent.putExtra(RecognizerIntent.EXTRA_SEGMENTED_SESSION, RecognizerIntent.EXTRA_AUDIO_SOURCE)
|
||||
}
|
||||
|
||||
@RequiresApi(Build.VERSION_CODES.TIRAMISU)
|
||||
private fun applySilenceSegmentedExtras(intent: Intent) {
|
||||
intent.putExtra(
|
||||
RecognizerIntent.EXTRA_SEGMENTED_SESSION,
|
||||
RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS,
|
||||
)
|
||||
}
|
||||
|
||||
@SuppressLint("MissingPermission")
|
||||
private fun openPushToTalkAudioSource(): PushToTalkAudioSource {
|
||||
check(Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU)
|
||||
val minBufferSize =
|
||||
AudioRecord.getMinBufferSize(
|
||||
pushToTalkSampleRateHz,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT,
|
||||
)
|
||||
check(minBufferSize > 0) { "AudioRecord buffer unavailable" }
|
||||
|
||||
val pipe = ParcelFileDescriptor.createPipe()
|
||||
var recorder: AudioRecord? = null
|
||||
var writeStream: ParcelFileDescriptor.AutoCloseOutputStream? = null
|
||||
try {
|
||||
recorder =
|
||||
AudioRecord(
|
||||
MediaRecorder.AudioSource.VOICE_RECOGNITION,
|
||||
pushToTalkSampleRateHz,
|
||||
AudioFormat.CHANNEL_IN_MONO,
|
||||
AudioFormat.ENCODING_PCM_16BIT,
|
||||
minBufferSize * 2,
|
||||
)
|
||||
check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord initialization failed" }
|
||||
recorder.startRecording()
|
||||
check(recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) { "AudioRecord did not start" }
|
||||
val activeRecorder = checkNotNull(recorder)
|
||||
val activeWriteStream = ParcelFileDescriptor.AutoCloseOutputStream(pipe[1])
|
||||
writeStream = activeWriteStream
|
||||
val source = PushToTalkAudioSource(pipe[0], activeWriteStream, activeRecorder)
|
||||
source.pumpJob =
|
||||
gatewayWorkScope.launch(Dispatchers.IO) {
|
||||
val buffer = ByteArray(minBufferSize.coerceAtLeast(4_096))
|
||||
try {
|
||||
while (currentCoroutineContext().isActive) {
|
||||
val bytesRead = activeRecorder.read(buffer, 0, buffer.size)
|
||||
if (bytesRead <= 0) break
|
||||
_inputLevel.value =
|
||||
TalkAudioLevel.smoothed(_inputLevel.value, TalkAudioLevel.pcm16Level(buffer, bytesRead))
|
||||
activeWriteStream.write(buffer, 0, bytesRead)
|
||||
}
|
||||
} catch (err: IOException) {
|
||||
Log.d(tag, "PTT audio pipe closed: ${err.message}")
|
||||
} finally {
|
||||
source.finishFromPump()
|
||||
}
|
||||
}
|
||||
return source
|
||||
} catch (err: Throwable) {
|
||||
runCatching { recorder?.stop() }
|
||||
runCatching { recorder?.release() }
|
||||
runCatching { writeStream?.close() }
|
||||
if (writeStream == null) runCatching { pipe[1].close() }
|
||||
runCatching { pipe[0].close() }
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
private fun schedulePushToTalkRestart(
|
||||
delayMs: Long,
|
||||
advanceRung: Boolean,
|
||||
) {
|
||||
val captureId = activePttCaptureId ?: return
|
||||
if (pttReleaseCompletion != null) return
|
||||
val rung = pttRecognitionRung ?: return
|
||||
val firstCandidate =
|
||||
when (rung) {
|
||||
is PushToTalkRecognitionRung.RawAudioSegmented ->
|
||||
if (advanceRung) {
|
||||
PushToTalkRecognitionCandidate.SilenceSegmented
|
||||
} else {
|
||||
PushToTalkRecognitionCandidate.RawAudioSegmented
|
||||
}
|
||||
PushToTalkRecognitionRung.SilenceSegmented ->
|
||||
if (advanceRung) {
|
||||
PushToTalkRecognitionCandidate.RestartingSingleSession
|
||||
} else {
|
||||
PushToTalkRecognitionCandidate.SilenceSegmented
|
||||
}
|
||||
PushToTalkRecognitionRung.RestartingSingleSession ->
|
||||
PushToTalkRecognitionCandidate.RestartingSingleSession
|
||||
}
|
||||
commitPushToTalkLivePartial()
|
||||
closePushToTalkRung()
|
||||
restartJob?.cancel()
|
||||
restartJob =
|
||||
gatewayWorkScope.launch {
|
||||
delay(delayMs)
|
||||
mainHandler.post {
|
||||
if (activePttCaptureId != captureId || pttReleaseCompletion != null || stopRequested) return@post
|
||||
try {
|
||||
startPushToTalkRecognition(captureId, firstCandidate)
|
||||
} catch (err: Throwable) {
|
||||
_isListening.value = false
|
||||
setTalkFailure(nativeText("Talk failed: \$message", err.message ?: err::class.simpleName.orEmpty()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun closePushToTalkRung() {
|
||||
(pttRecognitionRung as? PushToTalkRecognitionRung.RawAudioSegmented)?.source?.close()
|
||||
pttRecognitionRung = null
|
||||
}
|
||||
|
||||
private fun commitPushToTalkLivePartial() {
|
||||
val partial = pttLivePartial.trim()
|
||||
if (partial.isNotEmpty()) {
|
||||
pttFinalSegments += partial
|
||||
}
|
||||
pttLivePartial = ""
|
||||
}
|
||||
|
||||
private fun startListeningInternal(markListening: Boolean) {
|
||||
val r = recognizer ?: return
|
||||
val intent =
|
||||
@@ -2153,6 +2465,18 @@ class TalkModeManager internal constructor(
|
||||
isFinal: Boolean,
|
||||
) {
|
||||
val trimmed = text.trim()
|
||||
if (activePttCaptureId != null) {
|
||||
if (trimmed.isNotEmpty()) {
|
||||
if (isFinal) {
|
||||
pttFinalSegments += trimmed
|
||||
pttLivePartial = ""
|
||||
} else {
|
||||
pttLivePartial = trimmed
|
||||
}
|
||||
lastHeardAtMs = SystemClock.elapsedRealtime()
|
||||
}
|
||||
return
|
||||
}
|
||||
if (_isSpeaking.value && interruptOnSpeech) {
|
||||
if (shouldInterrupt(trimmed)) {
|
||||
stopSpeaking()
|
||||
@@ -2188,13 +2512,19 @@ class TalkModeManager internal constructor(
|
||||
|
||||
private fun checkSilence(captureId: String) {
|
||||
if (!_isListening.value) return
|
||||
val transcript = lastTranscript.trim()
|
||||
val transcript =
|
||||
if (activePttCaptureId != null) {
|
||||
PushToTalkTranscriptMerger.merge(pttFinalSegments, pttLivePartial)
|
||||
} else {
|
||||
lastTranscript.trim()
|
||||
}
|
||||
if (transcript.isEmpty()) return
|
||||
val lastHeard = lastHeardAtMs ?: return
|
||||
val elapsed = SystemClock.elapsedRealtime() - lastHeard
|
||||
if (elapsed < silenceWindowMs) return
|
||||
if (activePttCaptureId != null) {
|
||||
if (pttAutoStopEnabled) {
|
||||
if (pttReleaseCompletion != null) return
|
||||
gatewayWorkScope.launch { endPushToTalk(captureId) }
|
||||
}
|
||||
return
|
||||
@@ -2285,14 +2615,78 @@ class TalkModeManager internal constructor(
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun awaitPushToTalkRelease(captureId: String) {
|
||||
if (activePttCaptureId != captureId) return
|
||||
restartJob?.cancel()
|
||||
restartJob = null
|
||||
val rung = pttRecognitionRung ?: return
|
||||
// onResults, onError, and onEndOfSegmentedSession normally arrive well under a second,
|
||||
// so typical release latency is unchanged. The five-second bound only caps pathological recognizers;
|
||||
// leaving early truncates final words, which is worse than waiting.
|
||||
pttReleaseCompletion?.let { existing ->
|
||||
awaitPushToTalkReleaseCompletion(existing, pushToTalkReleaseGraceMs)
|
||||
return
|
||||
}
|
||||
if (!_isListening.value || recognizer == null) return
|
||||
|
||||
val completion = CompletableDeferred<Unit>()
|
||||
pttReleaseCompletion = completion
|
||||
_isListening.value = false
|
||||
_inputLevel.value = 0f
|
||||
when (rung) {
|
||||
is PushToTalkRecognitionRung.RawAudioSegmented -> {
|
||||
rung.source.requestFinish()
|
||||
// EXTRA_AUDIO_SOURCE is optional: a service may ignore the pipe and run its own mic,
|
||||
// so closing our AudioRecord alone would leave it listening past release. stopListening
|
||||
// forces its endpointer; for pipe-consuming services it is redundant after EOF.
|
||||
runCatching { recognizer?.stopListening() }.onFailure { completion.complete(Unit) }
|
||||
}
|
||||
PushToTalkRecognitionRung.SilenceSegmented,
|
||||
PushToTalkRecognitionRung.RestartingSingleSession,
|
||||
-> runCatching { recognizer?.stopListening() }.onFailure { completion.complete(Unit) }
|
||||
}
|
||||
awaitPushToTalkReleaseCompletion(completion, pushToTalkReleaseGraceMs)
|
||||
if (pttReleaseCompletion === completion) {
|
||||
pttReleaseCompletion = null
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun drainPushToTalkReleaseBeforeBegin() {
|
||||
val deadline = SystemClock.elapsedRealtime() + pushToTalkReleaseDrainTimeoutMs
|
||||
while (true) {
|
||||
val remainingMs = deadline - SystemClock.elapsedRealtime()
|
||||
val release = pttReleaseCompletion
|
||||
if (release != null && remainingMs > 0) {
|
||||
awaitPushToTalkReleaseCompletion(release, remainingMs)
|
||||
}
|
||||
if (activePttCaptureId == null && pttReleaseCompletion == null) return
|
||||
if (SystemClock.elapsedRealtime() >= deadline) return
|
||||
yield()
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun awaitPushToTalkReleaseCompletion(
|
||||
completion: CompletableDeferred<Unit>,
|
||||
timeoutMs: Long,
|
||||
) {
|
||||
try {
|
||||
withTimeoutOrNull(timeoutMs) { completion.await() }
|
||||
} catch (err: CancellationException) {
|
||||
if (completion.isCancelled && currentCoroutineContext().isActive) return
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
private fun clearPushToTalkRecognition(captureId: String): ClearedPushToTalkCapture? {
|
||||
if (activePttCaptureId != captureId) return null
|
||||
val transcript = lastTranscript.trim()
|
||||
val transcript = PushToTalkTranscriptMerger.merge(pttFinalSegments, pttLivePartial)
|
||||
val completion = pttCompletion
|
||||
pttTimeoutJob?.cancel()
|
||||
pttTimeoutJob = null
|
||||
pttAutoStopEnabled = false
|
||||
pttCompletion = null
|
||||
pttReleaseCompletion?.cancel()
|
||||
pttReleaseCompletion = null
|
||||
activePttCaptureId = null
|
||||
_isListening.value = false
|
||||
listeningMode = false
|
||||
@@ -2300,8 +2694,12 @@ class TalkModeManager internal constructor(
|
||||
recognizer?.cancel()
|
||||
recognizer?.destroy()
|
||||
recognizer = null
|
||||
closePushToTalkRung()
|
||||
pttFinalSegments.clear()
|
||||
pttLivePartial = ""
|
||||
lastTranscript = ""
|
||||
lastHeardAtMs = null
|
||||
_inputLevel.value = 0f
|
||||
return ClearedPushToTalkCapture(transcript = transcript, completion = completion)
|
||||
}
|
||||
|
||||
@@ -2966,24 +3364,39 @@ class TalkModeManager internal constructor(
|
||||
}
|
||||
}
|
||||
|
||||
private val listener =
|
||||
private val listener = recognitionListener(captureId = null)
|
||||
|
||||
private fun recognitionListener(captureId: String?): RecognitionListener =
|
||||
object : RecognitionListener {
|
||||
override fun onReadyForSpeech(params: Bundle?) {
|
||||
if (!acceptRecognitionCallback(captureId)) return
|
||||
// Only a live listening session may claim the status; a speech-interrupt
|
||||
// recognizer readying during playback must not touch Thinking state.
|
||||
if (_isEnabled.value && _isListening.value) {
|
||||
if (activePttCaptureId != null && _isListening.value) {
|
||||
setStatus(nativeText("Listening (PTT)"))
|
||||
} else if (_isEnabled.value && _isListening.value) {
|
||||
setStatus(nativeText("Listening"))
|
||||
}
|
||||
}
|
||||
|
||||
override fun onBeginningOfSpeech() {}
|
||||
override fun onBeginningOfSpeech() {
|
||||
if (!acceptRecognitionCallback(captureId)) return
|
||||
}
|
||||
|
||||
override fun onRmsChanged(rmsdB: Float) {}
|
||||
override fun onRmsChanged(rmsdB: Float) {
|
||||
if (!acceptRecognitionCallback(captureId)) return
|
||||
if (activePttCaptureId != null && pttRecognitionRung !is PushToTalkRecognitionRung.RawAudioSegmented) {
|
||||
_inputLevel.value = ((rmsdB + 2f) / 12f).coerceIn(0f, 1f)
|
||||
}
|
||||
}
|
||||
|
||||
override fun onBufferReceived(buffer: ByteArray?) {}
|
||||
|
||||
override fun onEndOfSpeech() {
|
||||
if (!acceptRecognitionCallback(captureId)) return
|
||||
clearListenWatchdog()
|
||||
_inputLevel.value = 0f
|
||||
if (activePttCaptureId != null) return
|
||||
// Don't restart while a transcript is being processed — the recognizer
|
||||
// competing for audio resources kills AudioTrack PCM playback.
|
||||
if (!finalizeInFlight) {
|
||||
@@ -2992,8 +3405,38 @@ class TalkModeManager internal constructor(
|
||||
}
|
||||
|
||||
override fun onError(error: Int) {
|
||||
if (!acceptRecognitionCallback(captureId)) return
|
||||
if (stopRequested) return
|
||||
_isListening.value = false
|
||||
_inputLevel.value = 0f
|
||||
if (activePttCaptureId != null) {
|
||||
pttReleaseCompletion?.let {
|
||||
it.complete(Unit)
|
||||
return
|
||||
}
|
||||
if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) {
|
||||
setStatus(nativeText("Microphone permission required"))
|
||||
return
|
||||
}
|
||||
setStatus(
|
||||
when (error) {
|
||||
SpeechRecognizer.ERROR_AUDIO -> nativeText("Audio error")
|
||||
SpeechRecognizer.ERROR_CLIENT -> nativeText("Client error")
|
||||
SpeechRecognizer.ERROR_NETWORK -> nativeText("Network error")
|
||||
SpeechRecognizer.ERROR_NETWORK_TIMEOUT -> nativeText("Network timeout")
|
||||
SpeechRecognizer.ERROR_NO_MATCH -> nativeText("Listening (PTT)")
|
||||
SpeechRecognizer.ERROR_RECOGNIZER_BUSY -> nativeText("Recognizer busy")
|
||||
SpeechRecognizer.ERROR_SERVER -> nativeText("Server error")
|
||||
SpeechRecognizer.ERROR_SPEECH_TIMEOUT -> nativeText("Listening (PTT)")
|
||||
else -> nativeText("Speech error (\$error)", error)
|
||||
},
|
||||
)
|
||||
schedulePushToTalkRestart(
|
||||
delayMs = 600L,
|
||||
advanceRung = pttRecognitionRung !is PushToTalkRecognitionRung.RestartingSingleSession,
|
||||
)
|
||||
return
|
||||
}
|
||||
if (error == SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS) {
|
||||
setStatus(nativeText("Microphone permission required"))
|
||||
return
|
||||
@@ -3016,21 +3459,60 @@ class TalkModeManager internal constructor(
|
||||
}
|
||||
|
||||
override fun onResults(results: Bundle?) {
|
||||
if (!acceptRecognitionCallback(captureId)) return
|
||||
val list = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
|
||||
list.firstOrNull()?.let { handleTranscript(it, isFinal = true) }
|
||||
if (activePttCaptureId != null) {
|
||||
_isListening.value = false
|
||||
_inputLevel.value = 0f
|
||||
pttReleaseCompletion?.let {
|
||||
it.complete(Unit)
|
||||
return
|
||||
}
|
||||
schedulePushToTalkRestart(
|
||||
delayMs = pushToTalkRestartDelayMs,
|
||||
advanceRung = pttRecognitionRung !is PushToTalkRecognitionRung.RestartingSingleSession,
|
||||
)
|
||||
return
|
||||
}
|
||||
scheduleRestart()
|
||||
}
|
||||
|
||||
override fun onPartialResults(partialResults: Bundle?) {
|
||||
if (!acceptRecognitionCallback(captureId)) return
|
||||
val list = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
|
||||
list.firstOrNull()?.let { handleTranscript(it, isFinal = false) }
|
||||
}
|
||||
|
||||
override fun onSegmentResults(segmentResults: Bundle) {
|
||||
if (!acceptRecognitionCallback(captureId) || activePttCaptureId == null) return
|
||||
val list = segmentResults.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION).orEmpty()
|
||||
list.firstOrNull()?.let { handleTranscript(it, isFinal = true) }
|
||||
}
|
||||
|
||||
override fun onEndOfSegmentedSession() {
|
||||
if (!acceptRecognitionCallback(captureId) || activePttCaptureId == null) return
|
||||
_isListening.value = false
|
||||
_inputLevel.value = 0f
|
||||
pttReleaseCompletion?.let {
|
||||
it.complete(Unit)
|
||||
return
|
||||
}
|
||||
schedulePushToTalkRestart(
|
||||
delayMs = 180L,
|
||||
advanceRung = shouldAdvancePushToTalkRungAfterSegmentedSession(pttRecognitionRung?.candidate ?: return),
|
||||
)
|
||||
}
|
||||
|
||||
override fun onEvent(
|
||||
eventType: Int,
|
||||
params: Bundle?,
|
||||
) {}
|
||||
}
|
||||
|
||||
// SpeechRecognizer can post callbacks after destroy. Binding each listener to its
|
||||
// capture prevents a retired session from mutating normal Talk or the next PTT hold.
|
||||
private fun acceptRecognitionCallback(captureId: String?): Boolean = captureId == activePttCaptureId
|
||||
}
|
||||
|
||||
private fun JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject
|
||||
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
package ai.openclaw.app.voice
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Test
|
||||
|
||||
class PushToTalkRecognitionLadderTest {
|
||||
@Test
|
||||
fun api33StartsWithRawAudioThenFallsThroughInOrder() {
|
||||
assertEquals(
|
||||
listOf(
|
||||
PushToTalkRecognitionCandidate.RawAudioSegmented,
|
||||
PushToTalkRecognitionCandidate.SilenceSegmented,
|
||||
PushToTalkRecognitionCandidate.RestartingSingleSession,
|
||||
),
|
||||
pushToTalkRecognitionCandidates(supportsSegmentedRecognition = true, first = null),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun olderApisUseRestartingSingleSessionOnly() {
|
||||
assertEquals(
|
||||
listOf(PushToTalkRecognitionCandidate.RestartingSingleSession),
|
||||
pushToTalkRecognitionCandidates(supportsSegmentedRecognition = false, first = null),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun degradedHoldNeverClimbsBackUpTheLadder() {
|
||||
assertEquals(
|
||||
listOf(
|
||||
PushToTalkRecognitionCandidate.SilenceSegmented,
|
||||
PushToTalkRecognitionCandidate.RestartingSingleSession,
|
||||
),
|
||||
pushToTalkRecognitionCandidates(
|
||||
supportsSegmentedRecognition = true,
|
||||
first = PushToTalkRecognitionCandidate.SilenceSegmented,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun rawSegmentedSessionEndAdvancesToSilenceButSilenceSessionEndRearms() {
|
||||
assertEquals(
|
||||
true,
|
||||
shouldAdvancePushToTalkRungAfterSegmentedSession(
|
||||
PushToTalkRecognitionCandidate.RawAudioSegmented,
|
||||
),
|
||||
)
|
||||
assertEquals(
|
||||
false,
|
||||
shouldAdvancePushToTalkRungAfterSegmentedSession(
|
||||
PushToTalkRecognitionCandidate.SilenceSegmented,
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package ai.openclaw.app.voice
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Test
|
||||
|
||||
class PushToTalkTranscriptMergerTest {
|
||||
@Test
|
||||
fun joinsMultipleFinalSegmentsInOrder() {
|
||||
assertEquals(
|
||||
"first thought. second thought. third thought",
|
||||
PushToTalkTranscriptMerger.merge(
|
||||
finalSegments = listOf("first thought", "second thought", "third thought"),
|
||||
livePartial = null,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun preservesTerminalPunctuationWithoutDoublingIt() {
|
||||
assertEquals(
|
||||
"ready? yes! \"done.\" next",
|
||||
PushToTalkTranscriptMerger.merge(
|
||||
finalSegments = listOf("ready?", "yes!", "\"done.\"", "next"),
|
||||
livePartial = null,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun preservesLocalePunctuationWithoutInjectingAsciiSeparators() {
|
||||
assertEquals(
|
||||
"你好。 世界",
|
||||
PushToTalkTranscriptMerger.merge(
|
||||
finalSegments = listOf("你好。", "世界"),
|
||||
livePartial = null,
|
||||
),
|
||||
)
|
||||
assertEquals(
|
||||
"هل أنت جاهز؟ نعم",
|
||||
PushToTalkTranscriptMerger.merge(
|
||||
finalSegments = listOf("هل أنت جاهز؟", "نعم"),
|
||||
livePartial = null,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun omitsTrailingPartialThatDuplicatesLastFinal() {
|
||||
assertEquals(
|
||||
"first. SAME words",
|
||||
PushToTalkTranscriptMerger.merge(
|
||||
finalSegments = listOf("first", "SAME words"),
|
||||
livePartial = " same WORDS ",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun appendsDistinctTrailingPartial() {
|
||||
assertEquals(
|
||||
"first final. trailing words",
|
||||
PushToTalkTranscriptMerger.merge(
|
||||
finalSegments = listOf("first final"),
|
||||
livePartial = "trailing words",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun returnsPartialForPartialOnlyHold() {
|
||||
assertEquals(
|
||||
"unfinished thought",
|
||||
PushToTalkTranscriptMerger.merge(
|
||||
finalSegments = emptyList(),
|
||||
livePartial = " unfinished thought ",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun returnsEmptyForEmptyResults() {
|
||||
assertEquals(
|
||||
"",
|
||||
PushToTalkTranscriptMerger.merge(
|
||||
finalSegments = listOf("", " "),
|
||||
livePartial = " ",
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -11,8 +11,11 @@ import ai.openclaw.app.i18n.verbatimText
|
||||
import android.Manifest
|
||||
import android.content.ComponentName
|
||||
import android.content.IntentFilter
|
||||
import android.os.Bundle
|
||||
import android.os.SystemClock
|
||||
import android.speech.RecognitionListener
|
||||
import android.speech.RecognitionService
|
||||
import android.speech.SpeechRecognizer
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
import kotlinx.coroutines.CoroutineDispatcher
|
||||
import kotlinx.coroutines.CoroutineScope
|
||||
@@ -26,6 +29,7 @@ import kotlinx.coroutines.async
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.test.StandardTestDispatcher
|
||||
import kotlinx.coroutines.test.advanceTimeBy
|
||||
import kotlinx.coroutines.test.advanceUntilIdle
|
||||
import kotlinx.coroutines.test.currentTime
|
||||
import kotlinx.coroutines.test.resetMain
|
||||
@@ -279,6 +283,147 @@ class TalkModeManagerTest {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun segmentDuringPushToTalkReleaseWaitsForEndOfSegmentedSession() {
|
||||
val manager = createManager()
|
||||
val releaseCompletion = CompletableDeferred<Unit>()
|
||||
setPrivateField(manager, "activePttCaptureId", "capture-1")
|
||||
setPrivateField(manager, "pttReleaseCompletion", releaseCompletion)
|
||||
val listener = recognitionListener(manager, "capture-1")
|
||||
val segment =
|
||||
Bundle().apply {
|
||||
putStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION, arrayListOf("first segment"))
|
||||
}
|
||||
|
||||
listener.onSegmentResults(segment)
|
||||
|
||||
assertFalse(releaseCompletion.isCompleted)
|
||||
assertEquals(listOf("first segment"), readPrivateField(manager, "pttFinalSegments"))
|
||||
|
||||
listener.onEndOfSegmentedSession()
|
||||
|
||||
assertTrue(releaseCompletion.isCompleted)
|
||||
}
|
||||
|
||||
@Test
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
fun releaseKeepsWaitingPastOldGraceForLateTerminalSegment() =
|
||||
runTest {
|
||||
val manager = createManager(isConnected = { false })
|
||||
val releaseCompletion = CompletableDeferred<Unit>()
|
||||
setPrivateField(manager, "activePttCaptureId", "capture-1")
|
||||
setPrivateField(manager, "pttReleaseCompletion", releaseCompletion)
|
||||
setPrivateField(manager, "pttRecognitionRung", silenceSegmentedRung())
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
(readPrivateField(manager, "pttFinalSegments") as MutableList<String>) += "early segment"
|
||||
val listener = recognitionListener(manager, "capture-1")
|
||||
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
|
||||
try {
|
||||
val ending = async { manager.endPushToTalk("capture-1") }
|
||||
runCurrent()
|
||||
|
||||
advanceTimeBy(1_200)
|
||||
listener.onSegmentResults(
|
||||
Bundle().apply {
|
||||
putStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION, arrayListOf("late segment"))
|
||||
},
|
||||
)
|
||||
assertFalse(ending.isCompleted)
|
||||
|
||||
listener.onEndOfSegmentedSession()
|
||||
advanceUntilIdle()
|
||||
|
||||
assertEquals("early segment. late segment", ending.await().transcript)
|
||||
} finally {
|
||||
Dispatchers.resetMain()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
fun cancelledEndPushToTalkClearsPendingReleaseBeforeNextBegin() =
|
||||
runTest {
|
||||
val app = RuntimeEnvironment.getApplication()
|
||||
shadowOf(app).grantPermissions(Manifest.permission.RECORD_AUDIO)
|
||||
val packageManager = shadowOf(app.packageManager)
|
||||
val speechService = ComponentName(app, "TestSpeechRecognitionService")
|
||||
packageManager.addServiceIfNotPresent(speechService)
|
||||
packageManager.addIntentFilterForService(speechService, IntentFilter(RecognitionService.SERVICE_INTERFACE))
|
||||
val manager = createManager()
|
||||
setPrivateField(manager, "activePttCaptureId", "capture-a")
|
||||
setPrivateField(manager, "pttReleaseCompletion", CompletableDeferred<Unit>())
|
||||
setPrivateField(manager, "pttRecognitionRung", silenceSegmentedRung())
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
(readPrivateField(manager, "pttFinalSegments") as MutableList<String>) += "capture a"
|
||||
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
|
||||
try {
|
||||
val ending = async { manager.endPushToTalk("capture-a") }
|
||||
runCurrent()
|
||||
ending.cancel()
|
||||
runCurrent()
|
||||
ending.join()
|
||||
|
||||
assertTrue(ending.isCancelled)
|
||||
assertNull(readPrivateField(manager, "activePttCaptureId"))
|
||||
assertNull(readPrivateField(manager, "pttReleaseCompletion"))
|
||||
assertEquals(emptyList<String>(), readPrivateField(manager, "pttFinalSegments"))
|
||||
|
||||
val started = manager.beginPushToTalk(allowNewCapture = true)
|
||||
|
||||
assertEquals(started.captureId, readPrivateField(manager, "activePttCaptureId"))
|
||||
assertEquals(emptyList<String>(), readPrivateField(manager, "pttFinalSegments"))
|
||||
} finally {
|
||||
manager.stopAllCapture()
|
||||
Dispatchers.resetMain()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@OptIn(ExperimentalCoroutinesApi::class)
|
||||
fun replacementBeginDrainsPendingReleaseBeforeStartingNewCapture() =
|
||||
runTest {
|
||||
val app = RuntimeEnvironment.getApplication()
|
||||
shadowOf(app).grantPermissions(Manifest.permission.RECORD_AUDIO)
|
||||
val packageManager = shadowOf(app.packageManager)
|
||||
val speechService = ComponentName(app, "TestSpeechRecognitionService")
|
||||
packageManager.addServiceIfNotPresent(speechService)
|
||||
packageManager.addIntentFilterForService(speechService, IntentFilter(RecognitionService.SERVICE_INTERFACE))
|
||||
var connectionChecks = 0
|
||||
val manager =
|
||||
createManager(
|
||||
isConnected = {
|
||||
connectionChecks += 1
|
||||
connectionChecks != 2
|
||||
},
|
||||
)
|
||||
val releaseCompletion = CompletableDeferred<Unit>()
|
||||
setPrivateField(manager, "activePttCaptureId", "capture-a")
|
||||
setPrivateField(manager, "pttReleaseCompletion", releaseCompletion)
|
||||
setPrivateField(manager, "pttRecognitionRung", silenceSegmentedRung())
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
(readPrivateField(manager, "pttFinalSegments") as MutableList<String>) += "first segment"
|
||||
Dispatchers.setMain(StandardTestDispatcher(testScheduler))
|
||||
try {
|
||||
val ending = async { manager.endPushToTalk("capture-a") }
|
||||
runCurrent()
|
||||
val starting = async { manager.beginPushToTalk(allowNewCapture = true) }
|
||||
runCurrent()
|
||||
|
||||
releaseCompletion.complete(Unit)
|
||||
advanceUntilIdle()
|
||||
|
||||
val ended = ending.await()
|
||||
val started = starting.await()
|
||||
assertEquals("offline", ended.status)
|
||||
assertEquals("first segment", ended.transcript)
|
||||
assertEquals(started.captureId, readPrivateField(manager, "activePttCaptureId"))
|
||||
assertEquals(emptyList<String>(), readPrivateField(manager, "pttFinalSegments"))
|
||||
} finally {
|
||||
manager.stopAllCapture()
|
||||
Dispatchers.resetMain()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun duplicateFinalForPendingTalkRunDoesNotStartAllResponseTts() {
|
||||
val manager = createManager()
|
||||
@@ -1125,6 +1270,20 @@ class TalkModeManagerTest {
|
||||
return method.invoke(manager, length) as Boolean
|
||||
}
|
||||
|
||||
private fun recognitionListener(
|
||||
manager: TalkModeManager,
|
||||
captureId: String,
|
||||
): RecognitionListener {
|
||||
val method = manager.javaClass.getDeclaredMethod("recognitionListener", String::class.java)
|
||||
method.isAccessible = true
|
||||
return method.invoke(manager, captureId) as RecognitionListener
|
||||
}
|
||||
|
||||
private fun silenceSegmentedRung(): Any {
|
||||
val clazz = Class.forName("ai.openclaw.app.voice.PushToTalkRecognitionRung\$SilenceSegmented")
|
||||
return requireNotNull(clazz.getField("INSTANCE").get(null))
|
||||
}
|
||||
|
||||
private fun chatFinalPayload(
|
||||
runId: String,
|
||||
text: String,
|
||||
|
||||
Reference in New Issue
Block a user