diff --git a/.env.example b/.env.example index 696d3a0617..67b2bc9d5f 100644 --- a/.env.example +++ b/.env.example @@ -51,6 +51,7 @@ RELAY_URL=ws://localhost:3000 # (use `just web` for Vite HMR instead). # BUZZ_WEB_DIR=./web/dist +# ----------------------------------------------------------------------------- # ----------------------------------------------------------------------------- # Git (NIP-34 bare repositories) # ----------------------------------------------------------------------------- diff --git a/crates/buzz-relay/src/config.rs b/crates/buzz-relay/src/config.rs index eaa67a052d..5852ce385f 100644 --- a/crates/buzz-relay/src/config.rs +++ b/crates/buzz-relay/src/config.rs @@ -120,7 +120,6 @@ pub struct Config { pub media_max_concurrent_uploads_per_pubkey: u32, /// Maximum media upload starts accepted from one pubkey per minute. pub media_uploads_per_minute: u32, - /// Optional override for ephemeral channel TTL (in seconds). /// When set, any channel created with a TTL tag will use this value instead /// of the client-provided one. Useful for testing ephemeral expiry quickly. @@ -184,7 +183,7 @@ impl Config { let bind_addr = parse_bind_addr(&bind_addr_raw)?; let database_url = std::env::var("DATABASE_URL") - .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); // sadscan:disable np.postgres.1 let redis_url = std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost:6379".to_string()); diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index a674e6d7c9..ac3fc3fd39 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -303,6 +303,7 @@ pub struct AppState { /// Per-uploader sliding-window rate limiter for media upload starts. /// Key: uploader pubkey bytes (32). Value: (count, window_start). pub media_upload_rate_limiter: Arc>, + /// Per-pubkey sliding-window rate limiter for transcription session minting. /// Current in-flight media uploads per uploader pubkey. pub media_uploads_in_flight: Arc>, /// Cache for observer agent-owner authorization (kind 24200). diff --git a/desktop/scripts/check-file-sizes.mjs b/desktop/scripts/check-file-sizes.mjs index d03e67e2b1..310bf065a0 100644 --- a/desktop/scripts/check-file-sizes.mjs +++ b/desktop/scripts/check-file-sizes.mjs @@ -255,7 +255,10 @@ const overrides = new Map([ // + mount-only useEffect for the Drafts-panel "Send message" confirm-dialog // flow. Load-bearing feature growth; queued to split with the rest of this // list. - ["src/features/messages/ui/MessageComposer.tsx", 1033], + // +15 local-dictation integration: useLocalDictation wiring (mic toggle, + // transcript-append handler, recording/transcribing state, stop-on-send/edit + // guards). Load-bearing feature growth; queued to split with the rest. + ["src/features/messages/ui/MessageComposer.tsx", 1049], ]); await runFileSizeCheck({ diff --git a/desktop/src-tauri/src/app_state.rs b/desktop/src-tauri/src/app_state.rs index dca627fe6e..2254c5b790 100644 --- a/desktop/src-tauri/src/app_state.rs +++ b/desktop/src-tauri/src/app_state.rs @@ -9,6 +9,7 @@ use tauri::{AppHandle, Manager}; #[cfg(feature = "mesh-llm")] use tokio::sync::Mutex as AsyncMutex; +use crate::dictation::DictationState; use crate::huddle::HuddleState; use crate::managed_agents::config_bridge::SessionConfigCache; use crate::managed_agents::ManagedAgentProcess; @@ -22,11 +23,7 @@ pub struct AppState { pub channel_templates_store_lock: Mutex<()>, pub managed_agent_processes: Mutex>, pub huddle_state: Mutex, - /// Tauri app handle — stored after setup so huddle commands can emit - /// `huddle-state-changed` events without needing the handle threaded - /// through every call site. - /// - /// Set once during `setup()` in `lib.rs`; never cleared. + /// Tauri app handle — set once during `setup()`, never cleared. pub app_handle: Mutex>, /// Selected audio output device name. `None` = system default. /// Used by `connect_audio_relay` and TTS pipeline when opening sinks. @@ -46,6 +43,8 @@ pub struct AppState { /// listener is up before any restore/create can request a connection. #[cfg(feature = "mesh-llm")] pub mesh_coordinator: AsyncMutex>, + /// Local dictation state (Parakeet STT for composer voice input). + pub dictation_state: Mutex, } /// Parse the `BUZZ_PRIVATE_KEY` env var into identity keys. `Some` means the @@ -108,6 +107,7 @@ pub fn build_app_state() -> AppState { mesh_llm_runtime: AsyncMutex::new(None), #[cfg(feature = "mesh-llm")] mesh_coordinator: AsyncMutex::new(None), + dictation_state: Mutex::new(DictationState::new()), } } diff --git a/desktop/src-tauri/src/dictation.rs b/desktop/src-tauri/src/dictation.rs new file mode 100644 index 0000000000..b5351caa44 --- /dev/null +++ b/desktop/src-tauri/src/dictation.rs @@ -0,0 +1,276 @@ +//! Local dictation pipeline — uses the Parakeet STT engine for offline +//! speech-to-text in the message composer. +//! +//! Unlike the huddle STT pipeline (which posts kind:9 events to the relay), +//! dictation emits transcribed text back to the frontend via Tauri events +//! so the composer can display it in real-time. +//! +//! Key differences from huddle STT: +//! - No TTS barge-in / echo gating (no agent voice in composer context) +//! - No PTT (dictation uses a toggle button, not push-to-talk) +//! - Slightly longer silence threshold for more coherent sentences +//! - Text goes to the frontend, not to the relay + +use std::sync::Arc; + +use tauri::{Emitter, State}; + +use crate::app_state::AppState; +use crate::huddle::models; +use crate::stt_engine::{ + SttEngine, SttEngineConfig, DEFAULT_MAX_SPEECH_SAMPLES, DICTATION_PARTIAL_FLUSH_SAMPLES, + DICTATION_SILENCE_FLUSH_FRAMES, +}; + +/// Tauri event name emitted when a dictation transcript segment is ready. +const DICTATION_TRANSCRIPT_EVENT: &str = "dictation-transcript"; + +/// Tauri event name emitted when dictation state changes (started/stopped). +const DICTATION_STATE_EVENT: &str = "dictation-state"; + +/// State for the active dictation session. +/// +/// Stored in `AppState` behind a `Mutex`. Only one dictation session can be +/// active at a time (starting a new one stops the previous). +pub(crate) struct DictationState { + /// The running STT engine, if dictation is active. + engine: Option>, + /// Monotonically increasing session counter. Included in all emitted events + /// so the frontend can ignore stale transcripts from a previous session's + /// forwarder that arrive after a new session has started. + session_id: u64, +} + +impl DictationState { + pub fn new() -> Self { + Self { + engine: None, + session_id: 0, + } + } +} + +/// `start_dictation` — begin local STT dictation. +/// +/// Starts the Parakeet STT engine and spawns a task that emits +/// `dictation-transcript` events to the frontend as text is recognized. +/// Returns an error if models are not downloaded yet. +#[tauri::command] +pub async fn start_dictation(state: State<'_, AppState>) -> Result { + // Check if models are ready. + if !models::is_stt_ready() { + // Kick off download if not already in progress. + if let Some(mgr) = models::global_model_manager() { + mgr.start_stt_download(state.http_client.clone()); + } + return Err("STT model not ready — download in progress".to_string()); + } + + let model_dir = models::stt_model_dir().ok_or("STT model directory not found")?; + + // Stop any existing dictation session first. + stop_dictation_inner(&state, None); + + let config = SttEngineConfig { + model_dir, + silence_flush_frames: DICTATION_SILENCE_FLUSH_FRAMES, + max_speech_samples: DEFAULT_MAX_SPEECH_SAMPLES, + partial_flush_samples: Some(DICTATION_PARTIAL_FLUSH_SAMPLES), + tts_active: None, + tts_cancel: None, + ptt_active: None, + }; + + let (engine, text_rx) = SttEngine::new(config)?; + let engine = Arc::new(engine); + + // Store the engine in state and increment the session counter. + let session_id = { + let mut ds = state + .dictation_state + .lock() + .unwrap_or_else(|e| e.into_inner()); + ds.engine = Some(Arc::clone(&engine)); + ds.session_id += 1; + ds.session_id + }; + + // Spawn a task that forwards transcribed text to the frontend. + let app_handle = state + .app_handle + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(); + + if let Some(handle) = app_handle { + let _ = handle.emit( + DICTATION_STATE_EVENT, + serde_json::json!({ "state": "started", "session": session_id }), + ); + spawn_dictation_forwarder(text_rx, handle, session_id); + } + + Ok(session_id) +} + +/// `stop_dictation` — stop the active dictation session. +/// +/// The final transcript (if any) is emitted asynchronously by the forwarder +/// task. The `dictation-state: stopped` event is emitted by the forwarder +/// after all pending transcripts have been forwarded, ensuring the frontend +/// receives the final text before the stopped signal. +/// +/// `session` scopes the stop to a specific session: the engine is only torn +/// down when the currently-stored `session_id` matches. This prevents a +/// delayed/fire-and-forget stop from an old session (e.g. one deferred behind +/// a final audio flush) from killing a *newer* session the user started in the +/// meantime. Pass `None` for an unconditional stop (used on cancel/unmount). +#[tauri::command] +pub fn stop_dictation(session: Option, state: State<'_, AppState>) -> Result<(), String> { + stop_dictation_inner(&state, session); + // Note: `stopped` is emitted by the forwarder task after draining all + // pending transcripts — not here. This avoids a race where the frontend + // sees `stopped` before the final transcript arrives. + Ok(()) +} + +/// `push_dictation_audio` — feed raw PCM bytes into the dictation pipeline. +/// +/// Expects a raw binary body: an 8-byte little-endian `u64` session header +/// followed by f32 LE samples at 48 kHz mono. The header scopes the push to a +/// specific session — bytes are only fed to the engine when the header matches +/// the currently-stored `session_id`. This prevents late audio from a +/// just-stopped session (whose final `flushAudioBatch()` chunks are still +/// arriving) from being accepted by a *newer* session the user started in the +/// meantime and transcribed into the new draft. +/// +/// If no dictation session is active, or the session header doesn't match the +/// active session, the bytes are silently discarded. +#[tauri::command] +pub fn push_dictation_audio( + request: tauri::ipc::Request<'_>, + state: State<'_, AppState>, +) -> Result<(), String> { + /// Size of the leading little-endian `u64` session header. + const SESSION_HEADER_BYTES: usize = 8; + /// Maximum IPC audio batch size (audio payload only, excluding the header): 100 KB. + const MAX_AUDIO_BATCH_BYTES: usize = 100 * 1024; + + match request.body() { + tauri::ipc::InvokeBody::Raw(bytes) => { + if bytes.len() < SESSION_HEADER_BYTES { + return Err(format!( + "audio batch too small: {} bytes (need at least {} for session header)", + bytes.len(), + SESSION_HEADER_BYTES + )); + } + let (header, audio) = bytes.split_at(SESSION_HEADER_BYTES); + if audio.len() > MAX_AUDIO_BATCH_BYTES { + return Err(format!( + "audio batch too large: {} bytes (max {})", + audio.len(), + MAX_AUDIO_BATCH_BYTES + )); + } + // `split_at` guarantees `header` is exactly `SESSION_HEADER_BYTES` long. + let session = u64::from_le_bytes( + header + .try_into() + .map_err(|_| "invalid session header".to_string())?, + ); + let ds = state + .dictation_state + .lock() + .unwrap_or_else(|e| e.into_inner()); + // Only feed audio tagged with the currently-active session. Late + // chunks from an old session are silently dropped. + if session == ds.session_id { + if let Some(ref engine) = ds.engine { + engine.push_audio(audio.to_vec())?; + } + } + Ok(()) + } + _ => Err("expected raw binary body".to_string()), + } +} + +/// `get_dictation_status` — check if local dictation is available and/or active. +#[tauri::command] +pub fn get_dictation_status(state: State<'_, AppState>) -> DictationStatus { + let model_ready = models::is_stt_ready(); + let is_active = state + .dictation_state + .lock() + .unwrap_or_else(|e| e.into_inner()) + .engine + .is_some(); + + DictationStatus { + available: model_ready, + active: is_active, + } +} + +/// Response for `get_dictation_status`. +#[derive(serde::Serialize, Clone)] +pub struct DictationStatus { + /// Whether the local STT model is downloaded and ready. + pub available: bool, + /// Whether a dictation session is currently active. + pub active: bool, +} + +// ── Internal helpers ────────────────────────────────────────────────────────── + +fn stop_dictation_inner(state: &AppState, session: Option) { + let old_engine = { + let mut ds = state + .dictation_state + .lock() + .unwrap_or_else(|e| e.into_inner()); + // Session-scoped stop: only tear down when the requested session matches + // the one currently stored. A `None` session stops unconditionally. + match session { + Some(requested) if requested != ds.session_id => None, + _ => ds.engine.take(), + } + }; + if let Some(engine) = old_engine { + engine.shutdown(); + // Drop outside the lock — thread join may block briefly. + drop(engine); + } +} + +/// Spawn an async task that reads transcribed text and emits Tauri events. +/// +/// Each event includes the `session` ID so the frontend can ignore stale +/// transcripts from a previous session's forwarder. When the channel closes +/// (engine stopped), the forwarder emits `dictation-state: stopped`. +fn spawn_dictation_forwarder( + mut text_rx: tokio::sync::mpsc::Receiver, + app_handle: tauri::AppHandle, + session_id: u64, +) { + tauri::async_runtime::spawn(async move { + while let Some(text) = text_rx.recv().await { + if text.is_empty() { + continue; + } + let payload = serde_json::json!({ "text": text, "session": session_id }); + if app_handle + .emit(DICTATION_TRANSCRIPT_EVENT, payload) + .is_err() + { + break; // App window closed. + } + } + // All transcripts forwarded — signal the frontend that dictation is done. + let _ = app_handle.emit( + DICTATION_STATE_EVENT, + serde_json::json!({ "state": "stopped", "session": session_id }), + ); + }); +} diff --git a/desktop/src-tauri/src/huddle/mod.rs b/desktop/src-tauri/src/huddle/mod.rs index 5499ccab4c..efcffca9e2 100644 --- a/desktop/src-tauri/src/huddle/mod.rs +++ b/desktop/src-tauri/src/huddle/mod.rs @@ -40,23 +40,8 @@ pub mod wire; // ── Shared utilities ────────────────────────────────────────────────────────── -/// Drain and discard all pending messages until shutdown or disconnect. -/// Shared by both the STT and TTS worker threads for graceful degradation -/// when model files are missing or initialization fails. -pub(super) fn drain_until_shutdown( - rx: std::sync::mpsc::Receiver, - shutdown: &std::sync::atomic::AtomicBool, -) { - loop { - if shutdown.load(std::sync::atomic::Ordering::Acquire) { - break; - } - match rx.recv_timeout(std::time::Duration::from_millis(100)) { - Ok(_) => continue, - Err(_) => break, - } - } -} +/// Re-export from `stt_engine` for backward compatibility with `tts.rs`. +pub(super) use crate::stt_engine::drain_until_shutdown; // ── Re-exports ──────────────────────────────────────────────────────────────── diff --git a/desktop/src-tauri/src/huddle/stt.rs b/desktop/src-tauri/src/huddle/stt.rs index 6f502ca72c..de2be1031e 100644 --- a/desktop/src-tauri/src/huddle/stt.rs +++ b/desktop/src-tauri/src/huddle/stt.rs @@ -1,12 +1,14 @@ //! Speech-to-Text pipeline for huddle voice transcription. //! -//! Mental model: +//! This is a thin wrapper around `crate::stt_engine::SttEngine` configured +//! with huddle-specific settings (TTS barge-in, PTT gating, huddle silence +//! threshold). //! //! ```text //! AudioWorklet (48 kHz f32 PCM) //! → push_audio_pcm (Tauri cmd) -//! → SttPipeline::push_audio [bounded sync_channel] -//! → stt_worker thread +//! → SttPipeline::push_audio +//! → SttEngine worker thread //! rubato: 48 kHz → 16 kHz mono //! earshot VAD: accumulate speech frames //! sherpa-onnx Parakeet TDT-CTC 110M: transcribe on silence @@ -14,57 +16,36 @@ //! → tokio task (start_stt_pipeline) //! builds kind:9 event → relay //! ``` -//! -//! The worker runs on a dedicated `std::thread` (not async) because -//! sherpa-onnx is CPU-bound and not Send-safe across await points. use std::{ path::PathBuf, - sync::{ - atomic::{AtomicBool, Ordering}, - mpsc::{self, Receiver, SyncSender}, - Arc, - }, - thread, - time::Duration, + sync::{atomic::AtomicBool, Arc}, }; use tokio::sync::mpsc as tokio_mpsc; -// ── Public pipeline handle ──────────────────────────────────────────────────── - -/// Bounded audio queue capacity. -/// 100 ms batches at 48 kHz ≈ 19 KB each → 50 slots ≈ 5 s / ~1 MB max backlog. -const AUDIO_QUEUE_DEPTH: usize = 50; +use crate::stt_engine::{ + SttEngine, SttEngineConfig, DEFAULT_MAX_SPEECH_SAMPLES, DEFAULT_SILENCE_FLUSH_FRAMES, +}; -/// Maximum speech buffer size: 30 seconds at 16 kHz. -/// Prevents OOM if VAD stays in speech mode (noisy environment). -const MAX_SPEECH_SAMPLES: usize = 16_000 * 30; +// ── Public pipeline handle ──────────────────────────────────────────────────── -/// Handle to the running STT pipeline. +/// Handle to the running huddle STT pipeline. /// +/// Wraps `SttEngine` with huddle-specific construction (TTS flags, PTT). /// Not Clone — wrap in `Arc` to share across threads. -/// -/// The text receiver (`tokio::sync::mpsc::Receiver`) is returned -/// separately from `new()` so the caller can move it directly into an async -/// task without holding a Mutex across await points. #[derive(Debug)] pub struct SttPipeline { - /// Send raw PCM bytes (f32 LE, 48 kHz mono) into the pipeline. - audio_tx: SyncSender>, - /// Signals the worker thread to stop. - shutdown: Arc, - /// Worker thread handle — taken on drop to join cleanly. - thread: Option>, + engine: SttEngine, } impl SttPipeline { - /// Spawn the pipeline thread. + /// Spawn the huddle STT pipeline. /// /// `tts_active` is a shared flag set by the TTS pipeline while audio is /// playing. The STT worker uses it to: /// - discard accumulated speech (echo prevention / barge-in gating) - /// - apply a 200 ms cooldown after TTS stops before re-enabling STT + /// - apply a cooldown after TTS stops before re-enabling STT /// - detect barge-in: speech onset during TTS → set `tts_cancel` /// /// `tts_cancel` (optional) is the TTS pipeline's cancel flag. When the STT @@ -76,492 +57,40 @@ impl SttPipeline { /// When `None`, the pipeline runs in continuous VAD mode. /// /// Returns `Err` only if the thread cannot be spawned (OS error). - /// If model files are missing, the worker logs and exits cleanly — - /// the pipeline handle is still returned but will never produce text. - /// - /// The `tokio::sync::mpsc::Receiver` is returned separately so the - /// caller can move it directly into an async task. This avoids holding a - /// `Mutex` across await points (which would block a Tokio worker - /// thread on every `recv_timeout` call). pub fn new( model_dir: PathBuf, tts_active: Arc, tts_cancel: Option>, ptt_active: Option>, ) -> Result<(Self, tokio_mpsc::Receiver), String> { - let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); - let (text_tx, text_rx) = tokio_mpsc::channel::(64); - let shutdown = Arc::new(AtomicBool::new(false)); - - let shutdown_worker = Arc::clone(&shutdown); - let tts_cancel_worker = tts_cancel.as_ref().map(Arc::clone); - let ptt_active_worker = ptt_active.as_ref().map(Arc::clone); - let handle = thread::Builder::new() - .name("stt-worker".into()) - .spawn(move || { - stt_worker( - model_dir, - audio_rx, - text_tx, - shutdown_worker, - tts_active, - tts_cancel_worker, - ptt_active_worker, - ) - }) - .map_err(|e| format!("failed to spawn stt-worker thread: {e}"))?; - - let pipeline = Self { - audio_tx, - shutdown, - thread: Some(handle), + let config = SttEngineConfig { + model_dir, + silence_flush_frames: DEFAULT_SILENCE_FLUSH_FRAMES, + max_speech_samples: DEFAULT_MAX_SPEECH_SAMPLES, + partial_flush_samples: None, + tts_active: Some(tts_active), + tts_cancel, + ptt_active, }; - Ok((pipeline, text_rx)) + + let (engine, text_rx) = SttEngine::new(config)?; + Ok((Self { engine }, text_rx)) } /// Signal the worker thread to stop. pub fn shutdown(&self) { - self.shutdown.store(true, Ordering::Release); + self.engine.shutdown(); } - /// Returns `true` if the worker thread has exited (init failure, crash, or normal exit). - /// Used by hot-start to detect dead pipelines and clear them for retry. + /// Returns `true` if the worker thread has exited. pub fn is_finished(&self) -> bool { - self.thread.as_ref().is_none_or(|h| h.is_finished()) + self.engine.is_finished() } - /// Feed raw PCM bytes into the pipeline. + /// Feed raw PCM bytes (f32 LE, 48 kHz mono) into the pipeline. /// - /// Non-blocking. Drops audio silently if the pipeline can't keep up — - /// better to lose frames than to stall the UI thread. + /// Non-blocking. Drops audio silently if the pipeline can't keep up. pub fn push_audio(&self, pcm_bytes: Vec) -> Result<(), String> { - // Reject non-4-byte-aligned input — would silently truncate in bytes_to_f32. - if !pcm_bytes.len().is_multiple_of(4) { - return Err(format!( - "audio input not 4-byte aligned ({} bytes) — expected f32 LE samples", - pcm_bytes.len() - )); - } - // Drop audio if the pipeline can't keep up — better than blocking the UI. - let _ = self.audio_tx.try_send(pcm_bytes); - Ok(()) - } -} - -impl Drop for SttPipeline { - fn drop(&mut self) { - // Signal the worker to stop. - self.shutdown.store(true, Ordering::Release); - // Dropping `audio_tx` (implicitly when self is dropped after this fn) - // unblocks the worker's recv_timeout loop. Join to ensure clean exit. - if let Some(thread) = self.thread.take() { - let _ = thread.join(); - } - } -} - -// ── Worker thread ───────────────────────────────────────────────────────────── - -/// How many 16 kHz samples of silence before we flush to STT. -/// 300 ms × 16 000 Hz / 256 samples-per-frame ≈ 19 frames. -/// Previous value (28 frames / 450 ms) felt sluggish in conversation. -const SILENCE_FLUSH_FRAMES: usize = 19; - -/// Consecutive VAD speech frames required before triggering barge-in during TTS. -/// 20 frames × 256 samples / 16 kHz ≈ 320 ms — must be long enough to filter -/// speaker-to-mic feedback (TTS audio bleeding through the mic) while still -/// catching real human interruptions. 80 ms (previous: 5 frames) was too -/// aggressive — laptop speakers without headphones triggered false barge-in -/// within the first word of TTS playback. -const BARGE_IN_DEBOUNCE_FRAMES: usize = 20; - -/// earshot requires exactly 256 samples per frame at 16 kHz. -const VAD_FRAME_SAMPLES: usize = 256; - -/// VAD probability threshold — above this is considered speech. -const VAD_THRESHOLD: f32 = 0.5; - -/// How long the worker waits on the audio channel before checking the shutdown flag. -const RECV_TIMEOUT: Duration = Duration::from_millis(50); - -/// 50 ms cooldown after TTS stops before STT re-enables. -/// Prevents the tail of TTS audio from being transcribed as speech. -/// Previous value (200 ms) was eating the first word when the user spoke -/// immediately after the agent finished. -const TTS_COOLDOWN: Duration = Duration::from_millis(50); - -/// Number of ONNX Runtime intra-op threads used by the offline recognizer. -/// -/// Held at 1 (conservative) until we have a local A/B on real huddle audio. -/// Sherpa-onnx's Parakeet example uses 2 and most published RTF numbers are -/// at 2 threads on x86_64 server class hardware, but the encoder runs only -/// on VAD chunk boundaries on a dedicated thread, so the threading knob -/// trades worker latency against potential oversubscription with the audio -/// worklet on small Macs (4-core Intel especially). Bump to 2 once the A/B -/// shows it's safe on the minimum-spec target. -const STT_NUM_THREADS: i32 = 1; - -fn stt_worker( - model_dir: PathBuf, - audio_rx: Receiver>, - text_tx: tokio_mpsc::Sender, - shutdown: Arc, - tts_active: Arc, - tts_cancel: Option>, - ptt_active: Option>, -) { - // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── - use rubato::{Fft, FixedSync, Resampler}; - - let mut resampler = match Fft::::new(48_000, 16_000, 1024, 2, 1, FixedSync::Input) { - Ok(r) => r, - Err(e) => { - eprintln!("buzz-desktop: STT resampler init failed: {e}"); - return; - } - }; - let chunk_in = resampler.input_frames_next(); - - // ── 2. Initialise earshot VAD ───────────────────────────────────────────── - use earshot::{DefaultPredictor, Detector}; - let mut vad = Detector::new(DefaultPredictor::new()); - - // ── 3. Initialise sherpa-onnx recognizer ───────────────────────────────── - // - // Parakeet TDT-CTC 110M ships as a single `model.int8.onnx` (CTC head) plus - // `tokens.txt`. sherpa-onnx infers the model family from which inner config - // has a `model` path set, so we don't need to set `model_type` explicitly. - // (See rust-api-examples/parakeet_tdt_ctc_simulate_streaming_microphone.rs - // in k2-fsa/sherpa-onnx.) - use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; - - let tokens_path = model_dir.join("tokens.txt"); - let model_path = model_dir.join("model.int8.onnx"); - if !tokens_path.exists() || !model_path.exists() { - eprintln!( - "buzz-desktop: STT model not found at {} — STT disabled", - model_dir.display() - ); - drain_until_shutdown(audio_rx, &shutdown); - return; - } - - let mut cfg = OfflineRecognizerConfig::default(); - cfg.model_config.nemo_ctc.model = Some(model_path.to_string_lossy().into_owned()); - cfg.model_config.tokens = Some(tokens_path.to_string_lossy().into_owned()); - cfg.model_config.num_threads = STT_NUM_THREADS; - // Explicit — defaults are not part of the API contract, and noisy debug - // logging in release builds would be expensive on every VAD chunk. - cfg.model_config.debug = false; - - let recognizer = match OfflineRecognizer::create(&cfg) { - Some(r) => r, - None => { - eprintln!("buzz-desktop: OfflineRecognizer::create returned None — STT disabled"); - drain_until_shutdown(audio_rx, &shutdown); - return; - } - }; - - // ── 4. Processing state ─────────────────────────────────────────────────── - // Leftover 48 kHz samples that didn't fill a full resampler chunk. - let mut input_buf_48k: Vec = Vec::with_capacity(chunk_in * 2); - // Leftover 16 kHz samples that didn't fill a full VAD frame. - let mut leftover_16k: Vec = Vec::new(); - // Accumulated speech frames (16 kHz). - let mut speech_buf: Vec = Vec::new(); - // Consecutive silence frame count. - let mut silence_frames: usize = 0; - // Whether we're currently in a speech segment. - let mut in_speech = false; - // Consecutive speech frames seen during TTS — used for barge-in debounce. - let mut barge_in_frames: usize = 0; - // Timestamp when TTS last stopped — used for the 200 ms cooldown. - let mut tts_stopped_at: Option = None; - - // ── 5. Main loop ────────────────────────────────────────────────────────── - let mut tts_was_active = false; - let mut ptt_was_active = ptt_active - .as_ref() - .is_some_and(|p| p.load(Ordering::Acquire)); - loop { - // Check shutdown flag before blocking. - if shutdown.load(Ordering::Acquire) { - break; - } - - // Track TTS transitions to set the cooldown timer. - let tts_now = tts_active.load(Ordering::Acquire); - if tts_was_active && !tts_now { - // TTS just stopped — record the timestamp for the cooldown window. - tts_stopped_at = Some(std::time::Instant::now()); - } - tts_was_active = tts_now; - - // Track PTT transitions — flush accumulated speech when key is released. - // The worklet stops sending frames when PTT is inactive, so the normal - // silence-accumulation flush path never runs. We must flush here on the - // active→inactive edge to avoid buffering speech across PTT presses. - if let Some(ref ptt) = ptt_active { - let ptt_now = ptt.load(Ordering::Acquire); - if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { - flush_to_stt(&speech_buf, &recognizer, &text_tx); - speech_buf.clear(); - silence_frames = 0; - in_speech = false; - } - ptt_was_active = ptt_now; - } - - // Use recv_timeout so we can periodically check the shutdown flag. - let bytes = match audio_rx.recv_timeout(RECV_TIMEOUT) { - Ok(b) => b, - Err(mpsc::RecvTimeoutError::Timeout) => continue, - Err(mpsc::RecvTimeoutError::Disconnected) => break, // Sender dropped. - }; - - // Drain any additional pending messages to batch-process. - let mut batch = vec![bytes]; - while let Ok(b) = audio_rx.try_recv() { - batch.push(b); - } - - for bytes in batch { - // Convert raw bytes to f32 samples (little-endian). - let samples_48k = bytes_to_f32(&bytes); - input_buf_48k.extend_from_slice(&samples_48k); - - // Resample in chunk_in-sized blocks. - while input_buf_48k.len() >= chunk_in { - let chunk: Vec = input_buf_48k.drain(..chunk_in).collect(); - let resampled = resample_chunk(&mut resampler, &chunk); - process_16k_samples( - &resampled, - &mut leftover_16k, - &mut vad, - &mut speech_buf, - &mut silence_frames, - &mut in_speech, - &mut barge_in_frames, - &recognizer, - &text_tx, - &tts_active, - tts_cancel.as_deref(), - &mut tts_stopped_at, - ptt_active.as_ref(), - ); - } - } - } - - // No final flush — leave_huddle/end_huddle emit lifecycle events before - // the STT worker exits, so a final flush would post a kind:9 message AFTER - // the user has "left." Losing the last partial utterance is acceptable. -} - -/// Resample a mono 48 kHz chunk to 16 kHz using rubato. -/// Returns the resampled samples (may be empty on error). -fn resample_chunk(resampler: &mut rubato::Fft, chunk_48k: &[f32]) -> Vec { - use audioadapter_buffers::direct::InterleavedSlice; - use rubato::Resampler; - - // rubato expects interleaved layout even for mono. - let input = match InterleavedSlice::new(chunk_48k, 1, chunk_48k.len()) { - Ok(a) => a, - Err(e) => { - eprintln!("buzz-desktop: STT resample input error: {e}"); - return Vec::new(); - } - }; - - match resampler.process(&input, 0, None) { - Ok(out) => out.take_data(), - Err(e) => { - eprintln!("buzz-desktop: STT resample error: {e}"); - Vec::new() - } + self.engine.push_audio(pcm_bytes) } } - -/// Feed 16 kHz samples through the VAD and accumulate speech. -/// Flushes to STT when silence exceeds threshold. -/// -/// When `tts_active` is set: -/// - In PTT mode: skip accumulation (PTT press handles TTS cancellation). -/// - In VAD mode: speech onset triggers barge-in via `tts_cancel`. -/// - After TTS stops, a cooldown prevents tail audio from being transcribed. -/// -/// When `ptt_active` is `Some`: -/// - VAD `is_speech` is ANDed with the PTT flag — when the key is released, -/// `is_speech` becomes false, silence_frames accumulates, and the existing -/// flush logic kicks in naturally. The 200 ms release delay + ~300 ms -/// silence flush gives a natural utterance tail. -#[allow(clippy::too_many_arguments)] -fn process_16k_samples( - samples: &[f32], - leftover: &mut Vec, - vad: &mut earshot::Detector, - speech_buf: &mut Vec, - silence_frames: &mut usize, - in_speech: &mut bool, - barge_in_frames: &mut usize, - recognizer: &sherpa_onnx::OfflineRecognizer, - text_tx: &tokio_mpsc::Sender, - tts_active: &Arc, - tts_cancel: Option<&AtomicBool>, - tts_stopped_at: &mut Option, - ptt_active: Option<&Arc>, -) { - leftover.extend_from_slice(samples); - - while leftover.len() >= VAD_FRAME_SAMPLES { - let frame: Vec = leftover.drain(..VAD_FRAME_SAMPLES).collect(); - let clamped: Vec = frame.iter().map(|&s| s.clamp(-1.0, 1.0)).collect(); - let prob = vad.predict_f32(&clamped); - let is_speech = prob > VAD_THRESHOLD; - - // PTT gating: when PTT key is not held, treat as silence. - // This causes natural flush when the key is released — silence_frames - // accumulates and the existing flush logic kicks in after - // SILENCE_FLUSH_FRAMES. The 200 ms release delay + ~300 ms silence - // flush gives a natural utterance tail. - let is_speech = if let Some(ptt) = ptt_active { - is_speech && ptt.load(Ordering::Acquire) - } else { - is_speech - }; - - let tts_playing = tts_active.load(Ordering::Acquire); - - // While TTS is playing: skip accumulation (echo prevention). - if tts_playing { - if ptt_active.is_some() { - // PTT mode — PTT press handles TTS cancellation directly - // (via the global shortcut handler). Just skip accumulation. - *in_speech = false; - *barge_in_frames = 0; - speech_buf.clear(); - *silence_frames = 0; - continue; - } - - // VAD mode — barge-in detection. - // Without acoustic echo cancellation, this requires a longer - // debounce (BARGE_IN_DEBOUNCE_FRAMES ≈ 320 ms) to filter - // speaker-to-mic feedback. - if is_speech { - *barge_in_frames += 1; - if *barge_in_frames >= BARGE_IN_DEBOUNCE_FRAMES { - // Real speech detected during TTS — trigger barge-in. - if let Some(cancel) = tts_cancel { - cancel.store(true, Ordering::Release); - } - *barge_in_frames = 0; - } - } else { - *barge_in_frames = 0; - } - // Don't accumulate speech during TTS (echo prevention). - *in_speech = false; - speech_buf.clear(); - *silence_frames = 0; - continue; - } - - // TTS not playing — check cooldown window. - if let Some(stopped) = *tts_stopped_at { - if stopped.elapsed() < TTS_COOLDOWN { - // Still in cooldown — discard but keep tracking speech state. - if !is_speech { - *in_speech = false; - } - speech_buf.clear(); - *silence_frames = 0; - *barge_in_frames = 0; - continue; - } else { - // Cooldown expired — clear the timer and reset all segment state. - *tts_stopped_at = None; - *in_speech = false; - *silence_frames = 0; - *barge_in_frames = 0; - } - } - - if is_speech { - *silence_frames = 0; - *in_speech = true; - speech_buf.extend_from_slice(&frame); - - // OOM guard: flush and reset if the buffer exceeds 30 s of audio. - if speech_buf.len() >= MAX_SPEECH_SAMPLES { - flush_to_stt(speech_buf, recognizer, text_tx); - speech_buf.clear(); - *silence_frames = 0; - *in_speech = false; - } - } else if *in_speech { - // Still accumulate during brief silence gaps. - speech_buf.extend_from_slice(&frame); - *silence_frames += 1; - - // In PTT mode, don't flush on silence — accumulate the entire - // key-hold as one utterance. The PTT release edge in the main - // loop handles the flush. In VAD mode, flush after the silence - // threshold so each natural pause becomes a separate message. - if ptt_active.is_none() && *silence_frames >= SILENCE_FLUSH_FRAMES { - // End of utterance — transcribe. - flush_to_stt(speech_buf, recognizer, text_tx); - speech_buf.clear(); - *silence_frames = 0; - *in_speech = false; - } - } - // If not in speech and not accumulating, just discard the frame. - } -} - -/// Run sherpa-onnx on the accumulated speech buffer and send the text. -/// -/// Uses `blocking_send` because this runs on a `std::thread` (not async). -/// The tokio channel's `blocking_send` is safe to call from sync contexts. -fn flush_to_stt( - speech_buf: &[f32], - recognizer: &sherpa_onnx::OfflineRecognizer, - text_tx: &tokio_mpsc::Sender, -) { - if speech_buf.is_empty() { - return; - } - - let stream = recognizer.create_stream(); - stream.accept_waveform(16_000, speech_buf); - recognizer.decode(&stream); - - let text = stream - .get_result() - .map(|r| r.text.trim().to_string()) - .unwrap_or_default(); - - if !text.is_empty() { - if let Err(e) = text_tx.blocking_send(text) { - eprintln!("buzz-desktop: STT text channel closed: {e}"); - } - } -} - -/// Convert raw bytes (f32 LE) to f32 samples. -/// Caller should ensure `bytes.len() % 4 == 0`; extra bytes are silently truncated. -/// -/// Assumes little-endian — matches all current Tauri targets (macOS ARM64, -/// Windows/Linux x86). The JS AudioWorklet's Float32Array uses platform-native -/// byte order, which is LE on all supported platforms. -fn bytes_to_f32(bytes: &[u8]) -> Vec { - bytes - .chunks_exact(4) - .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) - .collect() -} - -// drain_until_shutdown lives in super (huddle/mod.rs) — shared with tts.rs. -use super::drain_until_shutdown; diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 341c99d7b1..ea80137922 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -2,6 +2,7 @@ mod app_state; mod archive; mod commands; mod deep_link; +mod dictation; mod event_sync; mod events; mod huddle; @@ -19,6 +20,7 @@ mod ptt_shortcut; mod relay; mod secret_store; mod shutdown; +mod stt_engine; mod templates; mod util; @@ -68,6 +70,28 @@ fn perform_sidebar_default_haptic() { #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { + // Disable macOS press-and-hold accent picker so that holding shortcut keys + // (e.g. ⌘D for dictation push-to-talk) doesn't trigger the character popup. + // Writes to the app's own domain — does not affect other apps. + #[cfg(target_os = "macos")] + { + // Use the bundle ID for production; dev builds use the .dev suffix. + let bundle_id = if cfg!(debug_assertions) { + "xyz.block.buzz.app.dev" + } else { + "xyz.block.buzz.app" + }; + let _ = std::process::Command::new("defaults") + .args([ + "write", + bundle_id, + "ApplePressAndHoldEnabled", + "-bool", + "false", + ]) + .output(); + } + let builder = tauri::Builder::default() .plugin(tauri_plugin_single_instance::init(|app, argv, _cwd| { // Focus the existing window when a duplicate instance launches. @@ -629,6 +653,10 @@ pub fn run() { archive::delete_save_subscription, archive::read_archived_events, is_auto_update_supported, + dictation::start_dictation, + dictation::stop_dictation, + dictation::push_dictation_audio, + dictation::get_dictation_status, ]) .build(tauri::generate_context!()) .expect("error while building tauri application"); diff --git a/desktop/src-tauri/src/stt_engine.rs b/desktop/src-tauri/src/stt_engine.rs new file mode 100644 index 0000000000..eefe950fae --- /dev/null +++ b/desktop/src-tauri/src/stt_engine.rs @@ -0,0 +1,567 @@ +//! Reusable Speech-to-Text engine backed by sherpa-onnx (Parakeet TDT-CTC 110M). +//! +//! This module extracts the core STT logic — resample, VAD, inference — into a +//! standalone component that can be instantiated by both the huddle pipeline and +//! the composer dictation feature with different configurations. +//! +//! ```text +//! Caller (48 kHz f32 PCM) +//! → SttEngine::push_audio [bounded sync_channel] +//! → stt_worker thread +//! rubato: 48 kHz → 16 kHz mono +//! earshot VAD: accumulate speech frames +//! sherpa-onnx Parakeet TDT-CTC 110M: transcribe on silence +//! → text_rx [tokio mpsc channel] +//! → caller's async task +//! ``` +//! +//! The worker runs on a dedicated `std::thread` (not async) because +//! sherpa-onnx is CPU-bound and not Send-safe across await points. + +use std::{ + path::PathBuf, + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::{self, Receiver, SyncSender}, + Arc, + }, + thread, + time::Duration, +}; + +use tokio::sync::mpsc as tokio_mpsc; + +// ── Configuration ───────────────────────────────────────────────────────────── + +/// Default silence frames before flush (~300ms at 16 kHz / 256 samples per frame). +/// Used by the huddle pipeline. +pub const DEFAULT_SILENCE_FLUSH_FRAMES: usize = 19; + +/// Silence frames for dictation (~400ms) — slightly longer for more coherent sentences. +pub const DICTATION_SILENCE_FLUSH_FRAMES: usize = 25; + +/// Periodic partial flush interval for dictation streaming (~2 seconds of speech +/// at 16 kHz). When speech exceeds this threshold without a silence gap, the engine +/// flushes a partial transcript so the user sees text appearing on-the-fly. +pub const DICTATION_PARTIAL_FLUSH_SAMPLES: usize = 16_000 * 2; + +/// Default maximum speech buffer: 30 seconds at 16 kHz. +pub const DEFAULT_MAX_SPEECH_SAMPLES: usize = 16_000 * 30; + +/// Configuration for the STT engine. +/// +/// Allows callers to tune VAD behavior and optionally wire in TTS/PTT flags +/// for huddle-specific features (barge-in, echo gating, push-to-talk). +#[derive(Clone)] +pub struct SttEngineConfig { + /// Path to the directory containing `model.int8.onnx` and `tokens.txt`. + pub model_dir: PathBuf, + /// Number of consecutive silence frames before flushing to STT. + /// ~300ms = 19 frames, ~400ms = 25 frames at 16 kHz / 256 samples per frame. + pub silence_flush_frames: usize, + /// Maximum speech buffer size in samples (OOM guard). + pub max_speech_samples: usize, + /// Optional: when set, the engine flushes a partial transcript every N samples + /// of continuous speech (even without a silence gap). This enables on-the-fly + /// transcription for dictation. Set to `None` for huddle (silence-only flush). + pub partial_flush_samples: Option, + /// Optional: shared flag set by TTS while audio is playing (echo prevention). + pub tts_active: Option>, + /// Optional: TTS cancel flag — set by STT on barge-in detection. + pub tts_cancel: Option>, + /// Optional: push-to-talk flag — when `Some`, speech is only accumulated + /// while the flag is true. + pub ptt_active: Option>, +} + +// ── Public engine handle ────────────────────────────────────────────────────── + +/// Bounded audio queue capacity. +/// 100 ms batches at 48 kHz ≈ 19 KB each → 50 slots ≈ 5 s / ~1 MB max backlog. +const AUDIO_QUEUE_DEPTH: usize = 50; + +/// Handle to the running STT engine. +/// +/// Not Clone — wrap in `Arc` to share across threads. +/// +/// The text receiver (`tokio::sync::mpsc::Receiver`) is returned +/// separately from `new()` so the caller can move it directly into an async +/// task without holding a Mutex across await points. +#[derive(Debug)] +pub(crate) struct SttEngine { + /// Send raw PCM bytes (f32 LE, 48 kHz mono) into the engine. + audio_tx: SyncSender>, + /// Signals the worker thread to stop. + shutdown: Arc, + /// Worker thread handle — taken on drop to join cleanly. + thread: Option>, +} + +impl SttEngine { + /// Spawn the engine worker thread. + /// + /// Returns `(Self, Receiver)`. The receiver yields transcribed text + /// segments. It is returned separately so the caller can move it into an + /// async task without holding a Mutex. + /// + /// Returns `Err` only if the thread cannot be spawned (OS error). + /// If model files are missing, the worker logs and exits cleanly — + /// the engine handle is still returned but will never produce text. + pub fn new(config: SttEngineConfig) -> Result<(Self, tokio_mpsc::Receiver), String> { + let (audio_tx, audio_rx) = mpsc::sync_channel::>(AUDIO_QUEUE_DEPTH); + let (text_tx, text_rx) = tokio_mpsc::channel::(64); + let shutdown = Arc::new(AtomicBool::new(false)); + + let shutdown_worker = Arc::clone(&shutdown); + let handle = thread::Builder::new() + .name("stt-engine".into()) + .spawn(move || { + stt_worker(config, audio_rx, text_tx, shutdown_worker); + }) + .map_err(|e| format!("failed to spawn stt-engine thread: {e}"))?; + + let engine = Self { + audio_tx, + shutdown, + thread: Some(handle), + }; + Ok((engine, text_rx)) + } + + /// Signal the worker thread to stop. + pub fn shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + } + + /// Returns `true` if the worker thread has exited (init failure, crash, or normal exit). + pub fn is_finished(&self) -> bool { + self.thread.as_ref().is_none_or(|h| h.is_finished()) + } + + /// Feed raw PCM bytes (f32 LE, 48 kHz mono) into the engine. + /// + /// Non-blocking. Drops audio silently if the engine can't keep up — + /// better to lose frames than to stall the caller. + pub fn push_audio(&self, pcm_bytes: Vec) -> Result<(), String> { + if !pcm_bytes.len().is_multiple_of(4) { + return Err(format!( + "audio input not 4-byte aligned ({} bytes) — expected f32 LE samples", + pcm_bytes.len() + )); + } + let _ = self.audio_tx.try_send(pcm_bytes); + Ok(()) + } +} + +impl Drop for SttEngine { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::Release); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +// ── Shared utility ──────────────────────────────────────────────────────────── + +/// Drain and discard all pending messages until shutdown or disconnect. +/// +/// Used by both the STT and TTS worker threads for graceful degradation +/// when model files are missing or initialization fails. +pub(crate) fn drain_until_shutdown(rx: std::sync::mpsc::Receiver, shutdown: &AtomicBool) { + loop { + if shutdown.load(Ordering::Acquire) { + break; + } + match rx.recv_timeout(Duration::from_millis(100)) { + Ok(_) => continue, + Err(_) => break, + } + } +} + +// ── Worker thread ───────────────────────────────────────────────────────────── + +/// Consecutive VAD speech frames required before triggering barge-in during TTS. +/// 20 frames × 256 samples / 16 kHz ≈ 320 ms — filters speaker-to-mic feedback. +const BARGE_IN_DEBOUNCE_FRAMES: usize = 20; + +/// earshot requires exactly 256 samples per frame at 16 kHz. +const VAD_FRAME_SAMPLES: usize = 256; + +/// VAD probability threshold — above this is considered speech. +const VAD_THRESHOLD: f32 = 0.5; + +/// How long the worker waits on the audio channel before checking the shutdown flag. +const RECV_TIMEOUT: Duration = Duration::from_millis(50); + +/// Cooldown after TTS stops before STT re-enables. +/// Prevents the tail of TTS audio from being transcribed as speech. +const TTS_COOLDOWN: Duration = Duration::from_millis(50); + +/// Number of ONNX Runtime intra-op threads used by the offline recognizer. +const STT_NUM_THREADS: i32 = 1; + +fn stt_worker( + config: SttEngineConfig, + audio_rx: Receiver>, + text_tx: tokio_mpsc::Sender, + shutdown: Arc, +) { + // ── 1. Initialise rubato resampler (48 kHz → 16 kHz, mono) ─────────────── + use rubato::{Fft, FixedSync, Resampler}; + + let mut resampler = match Fft::::new(48_000, 16_000, 1024, 2, 1, FixedSync::Input) { + Ok(r) => r, + Err(e) => { + eprintln!("buzz-desktop: STT resampler init failed: {e}"); + return; + } + }; + let chunk_in = resampler.input_frames_next(); + + // ── 2. Initialise earshot VAD ───────────────────────────────────────────── + use earshot::{DefaultPredictor, Detector}; + let mut vad = Detector::new(DefaultPredictor::new()); + + // ── 3. Initialise sherpa-onnx recognizer ───────────────────────────────── + use sherpa_onnx::{OfflineRecognizer, OfflineRecognizerConfig}; + + let tokens_path = config.model_dir.join("tokens.txt"); + let model_path = config.model_dir.join("model.int8.onnx"); + if !tokens_path.exists() || !model_path.exists() { + eprintln!( + "buzz-desktop: STT model not found at {} — STT disabled", + config.model_dir.display() + ); + drain_until_shutdown(audio_rx, &shutdown); + return; + } + + let mut cfg = OfflineRecognizerConfig::default(); + cfg.model_config.nemo_ctc.model = Some(model_path.to_string_lossy().into_owned()); + cfg.model_config.tokens = Some(tokens_path.to_string_lossy().into_owned()); + cfg.model_config.num_threads = STT_NUM_THREADS; + cfg.model_config.debug = false; + + let recognizer = match OfflineRecognizer::create(&cfg) { + Some(r) => r, + None => { + eprintln!("buzz-desktop: OfflineRecognizer::create returned None — STT disabled"); + drain_until_shutdown(audio_rx, &shutdown); + return; + } + }; + + // ── 4. Processing state ─────────────────────────────────────────────────── + let mut input_buf_48k: Vec = Vec::with_capacity(chunk_in * 2); + let mut leftover_16k: Vec = Vec::new(); + let mut speech_buf: Vec = Vec::new(); + let mut silence_frames: usize = 0; + let mut in_speech = false; + let mut barge_in_frames: usize = 0; + let mut tts_stopped_at: Option = None; + + // ── 5. Main loop ────────────────────────────────────────────────────────── + let has_tts = config.tts_active.is_some(); + let tts_active_flag = config + .tts_active + .unwrap_or_else(|| Arc::new(AtomicBool::new(false))); + let tts_cancel_flag = config.tts_cancel; + let ptt_active_flag = config.ptt_active; + + let mut tts_was_active = false; + let mut ptt_was_active = ptt_active_flag + .as_ref() + .is_some_and(|p| p.load(Ordering::Acquire)); + + loop { + if shutdown.load(Ordering::Acquire) { + break; + } + + // Track TTS transitions (only relevant when TTS flags are wired). + if has_tts { + let tts_now = tts_active_flag.load(Ordering::Acquire); + if tts_was_active && !tts_now { + tts_stopped_at = Some(std::time::Instant::now()); + } + tts_was_active = tts_now; + } + + // Track PTT transitions — flush on key release. + if let Some(ref ptt) = ptt_active_flag { + let ptt_now = ptt.load(Ordering::Acquire); + if ptt_was_active && !ptt_now && in_speech && !speech_buf.is_empty() { + flush_to_stt(&speech_buf, &recognizer, &text_tx); + speech_buf.clear(); + silence_frames = 0; + in_speech = false; + } + ptt_was_active = ptt_now; + } + + let bytes = match audio_rx.recv_timeout(RECV_TIMEOUT) { + Ok(b) => b, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + + // Drain any additional pending messages to batch-process. + let mut batch = vec![bytes]; + while let Ok(b) = audio_rx.try_recv() { + batch.push(b); + } + + for bytes in batch { + let samples_48k = bytes_to_f32(&bytes); + input_buf_48k.extend_from_slice(&samples_48k); + + while input_buf_48k.len() >= chunk_in { + let chunk: Vec = input_buf_48k.drain(..chunk_in).collect(); + let resampled = resample_chunk(&mut resampler, &chunk); + process_16k_samples( + &resampled, + &mut leftover_16k, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &mut barge_in_frames, + &recognizer, + &text_tx, + has_tts, + &tts_active_flag, + tts_cancel_flag.as_deref(), + &mut tts_stopped_at, + ptt_active_flag.as_ref(), + config.silence_flush_frames, + config.max_speech_samples, + config.partial_flush_samples, + ); + } + } + } + + // ── Drain any audio still queued at shutdown ────────────────────────────── + // The caller flushes its final PCM batch (via `push_dictation_audio`) right + // before calling `stop_dictation`, which sets the shutdown flag. Without + // draining here, the loop above would break on the flag and skip that + // enqueued batch, so the tail of speech would never reach `speech_buf` and + // the final flush below would only transcribe older audio — dropping the + // last words. Process everything still in the channel before flushing. + while let Ok(bytes) = audio_rx.try_recv() { + let samples_48k = bytes_to_f32(&bytes); + input_buf_48k.extend_from_slice(&samples_48k); + + while input_buf_48k.len() >= chunk_in { + let chunk: Vec = input_buf_48k.drain(..chunk_in).collect(); + let resampled = resample_chunk(&mut resampler, &chunk); + process_16k_samples( + &resampled, + &mut leftover_16k, + &mut vad, + &mut speech_buf, + &mut silence_frames, + &mut in_speech, + &mut barge_in_frames, + &recognizer, + &text_tx, + has_tts, + &tts_active_flag, + tts_cancel_flag.as_deref(), + &mut tts_stopped_at, + ptt_active_flag.as_ref(), + config.silence_flush_frames, + config.max_speech_samples, + config.partial_flush_samples, + ); + } + } + + // ── Final flush — transcribe any remaining speech on shutdown/disconnect ── + if !speech_buf.is_empty() { + flush_to_stt(&speech_buf, &recognizer, &text_tx); + } +} + +/// Resample a mono 48 kHz chunk to 16 kHz using rubato. +fn resample_chunk(resampler: &mut rubato::Fft, chunk_48k: &[f32]) -> Vec { + use audioadapter_buffers::direct::InterleavedSlice; + use rubato::Resampler; + + let input = match InterleavedSlice::new(chunk_48k, 1, chunk_48k.len()) { + Ok(a) => a, + Err(e) => { + eprintln!("buzz-desktop: STT resample input error: {e}"); + return Vec::new(); + } + }; + + match resampler.process(&input, 0, None) { + Ok(out) => out.take_data(), + Err(e) => { + eprintln!("buzz-desktop: STT resample error: {e}"); + Vec::new() + } + } +} + +/// Feed 16 kHz samples through the VAD and accumulate speech. +/// Flushes to STT when silence exceeds the configured threshold. +#[allow(clippy::too_many_arguments)] +fn process_16k_samples( + samples: &[f32], + leftover: &mut Vec, + vad: &mut earshot::Detector, + speech_buf: &mut Vec, + silence_frames: &mut usize, + in_speech: &mut bool, + barge_in_frames: &mut usize, + recognizer: &sherpa_onnx::OfflineRecognizer, + text_tx: &tokio_mpsc::Sender, + has_tts: bool, + tts_active: &Arc, + tts_cancel: Option<&AtomicBool>, + tts_stopped_at: &mut Option, + ptt_active: Option<&Arc>, + silence_flush_threshold: usize, + max_speech_samples: usize, + partial_flush_samples: Option, +) { + leftover.extend_from_slice(samples); + + while leftover.len() >= VAD_FRAME_SAMPLES { + let frame: Vec = leftover.drain(..VAD_FRAME_SAMPLES).collect(); + let clamped: Vec = frame.iter().map(|&s| s.clamp(-1.0, 1.0)).collect(); + let prob = vad.predict_f32(&clamped); + let is_speech = prob > VAD_THRESHOLD; + + // PTT gating: when PTT key is not held, treat as silence. + let is_speech = if let Some(ptt) = ptt_active { + is_speech && ptt.load(Ordering::Acquire) + } else { + is_speech + }; + + // TTS echo prevention (only when TTS flags are wired). + if has_tts { + let tts_playing = tts_active.load(Ordering::Acquire); + + if tts_playing { + if ptt_active.is_some() { + // PTT mode — skip accumulation. + *in_speech = false; + *barge_in_frames = 0; + speech_buf.clear(); + *silence_frames = 0; + continue; + } + + // VAD mode — barge-in detection. + if is_speech { + *barge_in_frames += 1; + if *barge_in_frames >= BARGE_IN_DEBOUNCE_FRAMES { + if let Some(cancel) = tts_cancel { + cancel.store(true, Ordering::Release); + } + *barge_in_frames = 0; + } + } else { + *barge_in_frames = 0; + } + *in_speech = false; + speech_buf.clear(); + *silence_frames = 0; + continue; + } + + // TTS cooldown window. + if let Some(stopped) = *tts_stopped_at { + if stopped.elapsed() < TTS_COOLDOWN { + if !is_speech { + *in_speech = false; + } + speech_buf.clear(); + *silence_frames = 0; + *barge_in_frames = 0; + continue; + } else { + *tts_stopped_at = None; + *in_speech = false; + *silence_frames = 0; + *barge_in_frames = 0; + } + } + } + + if is_speech { + *silence_frames = 0; + *in_speech = true; + speech_buf.extend_from_slice(&frame); + + // OOM guard. + if speech_buf.len() >= max_speech_samples { + flush_to_stt(speech_buf, recognizer, text_tx); + speech_buf.clear(); + *silence_frames = 0; + *in_speech = false; + } + // Periodic partial flush — emit intermediate transcript so text + // appears on-the-fly during continuous speech (dictation mode). + else if let Some(threshold) = partial_flush_samples { + if speech_buf.len() >= threshold { + flush_to_stt(speech_buf, recognizer, text_tx); + speech_buf.clear(); + // Stay in_speech — the user is still talking. + } + } + } else if *in_speech { + speech_buf.extend_from_slice(&frame); + *silence_frames += 1; + + // In PTT mode, don't flush on silence — the PTT release edge handles it. + if ptt_active.is_none() && *silence_frames >= silence_flush_threshold { + flush_to_stt(speech_buf, recognizer, text_tx); + speech_buf.clear(); + *silence_frames = 0; + *in_speech = false; + } + } + } +} + +/// Run sherpa-onnx on the accumulated speech buffer and send the text. +fn flush_to_stt( + speech_buf: &[f32], + recognizer: &sherpa_onnx::OfflineRecognizer, + text_tx: &tokio_mpsc::Sender, +) { + if speech_buf.is_empty() { + return; + } + + let stream = recognizer.create_stream(); + stream.accept_waveform(16_000, speech_buf); + recognizer.decode(&stream); + + let text = stream + .get_result() + .map(|r| r.text.trim().to_string()) + .unwrap_or_default(); + + if !text.is_empty() { + if let Err(e) = text_tx.blocking_send(text) { + eprintln!("buzz-desktop: STT text channel closed: {e}"); + } + } +} + +/// Convert raw bytes (f32 LE) to f32 samples. +fn bytes_to_f32(bytes: &[u8]) -> Vec { + bytes + .chunks_exact(4) + .map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])) + .collect() +} diff --git a/desktop/src/app/AppShell.tsx b/desktop/src/app/AppShell.tsx index e35c41c0d2..1afc74321f 100644 --- a/desktop/src/app/AppShell.tsx +++ b/desktop/src/app/AppShell.tsx @@ -527,6 +527,11 @@ export function AppShell() { return; } + // Track whether ⌘D keydown was dispatched so keyup only fires when + // a dictation session was actually started (avoids spurious events on + // normal 'd' typing). + let dictationKeyHeld = false; + function handleKeyDown(event: KeyboardEvent) { if (!hasPrimaryShortcutModifier(event) || event.altKey) { return; @@ -557,6 +562,13 @@ export function AppShell() { return; } + if (key === "d" && !event.shiftKey) { + event.preventDefault(); + dictationKeyHeld = true; + window.dispatchEvent(new CustomEvent("buzz:dictation-key-down")); + return; + } + if (key === "a" && event.shiftKey) { event.preventDefault(); void goHome(); @@ -564,9 +576,18 @@ export function AppShell() { } } + function handleKeyUp(event: KeyboardEvent) { + if (event.key.toLowerCase() === "d" && dictationKeyHeld) { + dictationKeyHeld = false; + window.dispatchEvent(new CustomEvent("buzz:dictation-key-up")); + } + } + window.addEventListener("keydown", handleKeyDown); + window.addEventListener("keyup", handleKeyUp); return () => { window.removeEventListener("keydown", handleKeyDown); + window.removeEventListener("keyup", handleKeyUp); }; }, [ handleOpenBrowseChannels, diff --git a/desktop/src/features/dictation/hooks/useComposerDictation.ts b/desktop/src/features/dictation/hooks/useComposerDictation.ts new file mode 100644 index 0000000000..fc613b9495 --- /dev/null +++ b/desktop/src/features/dictation/hooks/useComposerDictation.ts @@ -0,0 +1,157 @@ +import type * as React from "react"; +import { useEffect, useId, useRef } from "react"; +import { + clearActiveDictationComposer, + isActiveDictationComposer, + setActiveDictationComposer, +} from "../lib/activeComposer"; +import { useDictation } from "./useDictation"; + +interface UseComposerDictationOptions { + /** Ref to a function that syncs contentRef from the Tiptap editor and returns it. */ + syncContentRef: React.MutableRefObject<() => string>; + /** Whether the composer is currently disabled (read-only, etc.). */ + disabled?: boolean; + disabledRef: React.MutableRefObject; + isSendingRef: React.MutableRefObject; + isUploadingRef: React.MutableRefObject; + /** Updates contentRef + isContentEmpty state. */ + setComposerContent: (text: string) => void; + /** Ref to a function that updates the Tiptap editor document. */ + setEditorContentRef: React.MutableRefObject<(text: string) => void>; + submitMessageRef: React.MutableRefObject<() => void>; + /** When this key changes (channel/thread switch), active dictation is stopped. */ + draftKey?: string | null; + /** Ref to the composer's container element for focus tracking. */ + composerRef?: React.RefObject; +} + +/** + * Thin wrapper around `useDictation` pre-wired for the MessageComposer's + * state management (syncContentRef, setComposerContent, editor, submitMessageRef). + * + * Uses the local Parakeet STT engine — fully offline, no relay or API key needed. + */ +export function useComposerDictation({ + syncContentRef, + disabled = false, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContentRef, + submitMessageRef, + draftKey, + composerRef, +}: UseComposerDictationOptions) { + const instanceId = useId(); + const isSendBlockedRef = useRef(false); + isSendBlockedRef.current = + disabledRef.current || isSendingRef.current || isUploadingRef.current; + + const dictation = useDictation({ + getText: () => syncContentRef.current(), + setText: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + }, + onSend: (text) => { + setComposerContent(text); + setEditorContentRef.current(text); + // Submit synchronously — the content ref is already set above, so + // syncComposerContentFromEditor() will serialize the editor which now + // holds the dictated text. + submitMessageRef.current(); + }, + isSendBlockedRef, + }); + + // Track which composer is active (most recently focused) so that the global + // ⌘D shortcut only dispatches to one instance when multiple are mounted. + useEffect(() => { + const el = composerRef?.current; + if (!el) { + // If no ref provided, this composer is always considered active (single-composer case). + setActiveDictationComposer(instanceId); + return () => clearActiveDictationComposer(instanceId); + } + + function handleFocusIn() { + setActiveDictationComposer(instanceId); + } + + el.addEventListener("focusin", handleFocusIn); + return () => { + el.removeEventListener("focusin", handleFocusIn); + clearActiveDictationComposer(instanceId); + }; + }, [instanceId, composerRef]); + + // Cancel dictation when the channel/thread changes so that transcript events + // from a stale local STT session don't leak into the wrong draft. + // Only cancel if this instance owns a live/pending session — avoids killing + // another composer's session since the native engine is a singleton. + // + // Includes `isTranscribing`, not just `isRecording`/`isStarting`: + // `stopRecording()` clears `isRecording` immediately but deliberately keeps + // the transcript listener alive (isTranscribing=true) until the native + // `stopped` event, so the final local STT flush can still be appended. If the + // draftKey changes during that grace window, we must still cancel — otherwise + // the late transcript is appended through this same composer instance into the + // newly restored draft. `cancelRecording()` unlistens the transcript handler + // and performs a session-scoped native stop, so this is safe. + const isOwningSessionRef = useRef(false); + isOwningSessionRef.current = + dictation.isRecording || dictation.isStarting || dictation.isTranscribing; + // biome-ignore lint/correctness/useExhaustiveDependencies: draftKey is the sole trigger + useEffect(() => { + if (isOwningSessionRef.current) { + dictation.cancelRecording(); + } + }, [draftKey]); + + // Auto-cancel dictation when the composer becomes disabled mid-recording + // (e.g. channel becomes read-only, parent send state disables thread composer). + // Without this, the STT session keeps running with no way to stop it. + useEffect(() => { + if (disabled && dictation.isRecording) { + dictation.cancelRecording(); + } + }, [disabled, dictation.isRecording, dictation.cancelRecording]); + + // ⌘D push-to-talk — hold to record, release to stop. + // Dispatched from AppShell's keydown/keyup handlers. + // Only the active (most recently focused) composer responds, and only when + // not disabled/send-blocked. + // biome-ignore lint/correctness/useExhaustiveDependencies: disabledRef/isSendBlockedRef are stable refs read at call time + useEffect(() => { + function handleKeyDown() { + // Only respond if this is the active composer instance. + if (!isActiveDictationComposer(instanceId)) return; + // Don't start dictation in disabled/blocked composers. + if (disabledRef.current || isSendBlockedRef.current) return; + if (!dictation.isRecording && !dictation.isStarting) { + dictation.startRecording(); + } + } + function handleKeyUp() { + if (dictation.isRecording || dictation.isStarting) { + dictation.stopRecording(); + } + } + window.addEventListener("buzz:dictation-key-down", handleKeyDown); + window.addEventListener("buzz:dictation-key-up", handleKeyUp); + return () => { + window.removeEventListener("buzz:dictation-key-down", handleKeyDown); + window.removeEventListener("buzz:dictation-key-up", handleKeyUp); + }; + }, [ + instanceId, + dictation.isRecording, + dictation.isStarting, + dictation.startRecording, + dictation.stopRecording, + ]); + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useDictation.ts b/desktop/src/features/dictation/hooks/useDictation.ts new file mode 100644 index 0000000000..19aa3566cd --- /dev/null +++ b/desktop/src/features/dictation/hooks/useDictation.ts @@ -0,0 +1,86 @@ +import type * as React from "react"; +import { useCallback, useMemo, useRef } from "react"; +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "../lib/voiceInput"; +import { useLocalDictation } from "./useLocalDictation"; + +interface UseDictationOptions { + /** Returns the current composer text (must be fresh — synced from editor). */ + getText: () => string; + /** Set composer text */ + setText: (value: string) => void; + /** Send the message */ + onSend: (text: string) => void; + /** Ref that is `true` when sending is blocked (uploading, preparing mention, etc.) */ + isSendBlockedRef?: React.MutableRefObject; +} + +export function useDictation({ + getText, + setText, + onSend, + isSendBlockedRef, +}: UseDictationOptions) { + const autoSubmitPhrases = useMemo( + () => parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + [], + ); + const stopRecordingRef = useRef<() => void>(() => {}); + const lastTranscriptRef = useRef(""); + + const handleTranscript = useCallback( + (transcript: string) => { + const previous = lastTranscriptRef.current; + const latest = getText(); + const merged = replaceTrailingTranscribedText( + latest, + previous, + transcript, + ); + const match = getAutoSubmitMatch(transcript, autoSubmitPhrases); + + if (!match) { + setText(merged); + // Reset to empty — each streaming partial is an independent segment + // (the native engine flushes and clears its buffer). The next transcript + // should be appended, not replace this one. + lastTranscriptRef.current = ""; + return; + } + + const textWithoutPhrase = replaceTrailingTranscribedText( + latest, + previous, + match.textWithoutPhrase, + ); + if (!textWithoutPhrase.trim()) return; + + stopRecordingRef.current(); + + if (isSendBlockedRef?.current) { + setText(textWithoutPhrase); + return; + } + + setText(textWithoutPhrase.trim()); + onSend(textWithoutPhrase.trim()); + lastTranscriptRef.current = ""; + }, + [autoSubmitPhrases, getText, onSend, isSendBlockedRef, setText], + ); + + const dictation = useLocalDictation({ + onRecordingStart: () => { + lastTranscriptRef.current = ""; + }, + onTranscriptText: handleTranscript, + }); + + stopRecordingRef.current = dictation.stopRecording; + + return dictation; +} diff --git a/desktop/src/features/dictation/hooks/useLocalDictation.ts b/desktop/src/features/dictation/hooks/useLocalDictation.ts new file mode 100644 index 0000000000..0f8394551a --- /dev/null +++ b/desktop/src/features/dictation/hooks/useLocalDictation.ts @@ -0,0 +1,555 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen, type UnlistenFn } from "@tauri-apps/api/event"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; + +/** + * Raw binary invoke — uses Tauri's internal IPC for zero-copy ArrayBuffer transfer. + * Same pattern as huddle's audioWorklet.ts. + */ +function invokeRawBinary(cmd: string, payload: Uint8Array): Promise { + // biome-ignore lint/suspicious/noExplicitAny: Tauri internals have no public type definition + const internals = (window as any).__TAURI_INTERNALS__; + if (!internals?.invoke) { + return Promise.reject(new Error("Tauri internals not available")); + } + return internals.invoke(cmd, payload); +} + +interface UseLocalDictationOptions { + disabled?: boolean; + onRecordingStart?: () => void; + onTranscriptText: (text: string) => void; +} + +interface DictationStatus { + available: boolean; + active: boolean; +} + +const DICTATION_TRANSCRIPT_EVENT = "dictation-transcript"; +const DICTATION_STATE_EVENT = "dictation-state"; + +/** Interval (ms) to poll model availability after initial unavailability. */ +const MODEL_POLL_INTERVAL_MS = 5_000; + +/** + * Batching interval for audio IPC (ms). The worklet posts every render quantum + * (~2.67ms at 48kHz/128 samples). Sending each one individually overloads IPC. + * We accumulate ~100ms of audio before sending to reduce IPC overhead from + * ~375 calls/s to ~10 calls/s. + */ +const AUDIO_BATCH_MS = 100; + +/** + * Max samples per `push_dictation_audio` IPC call. The native command rejects + * any raw audio payload over 100 KB (`MAX_AUDIO_BATCH_BYTES` in `dictation.rs`); + * at 48 kHz f32 mono that is 25,600 samples (~0.53s). We chunk under that cap + * (24,000 samples / 96 KB, leaving headroom) so a stalled main thread that + * lets the batch grow past ~0.5s can't produce a single oversized buffer that + * native rejects and we silently drop. Chunks are sent in order. + */ +const MAX_IPC_SAMPLES = 24_000; + +/** + * Size (bytes) of the little-endian `u64` session header prepended to each + * `push_dictation_audio` payload. Native reads this header and only feeds audio + * whose session matches the currently-active one, so late chunks from a + * just-stopped session can't leak into a newer session's transcript. + */ +const SESSION_HEADER_BYTES = 8; + +/** + * Local STT dictation hook using the Parakeet model via Tauri native commands. + * + * Works fully offline — no relay or OpenAI API key needed. Uses the same + * sherpa-onnx Parakeet TDT-CTC 110M model as huddle transcription. + * + * Audio capture uses the Web Audio API (AudioWorklet) on the frontend side, + * then sends batched raw PCM bytes to the native STT engine via + * `push_dictation_audio`. + */ +export function useLocalDictation({ + disabled = false, + onRecordingStart, + onTranscriptText, +}: UseLocalDictationOptions) { + const [isRecording, setIsRecording] = useState(false); + const [isStarting, setIsStarting] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); + const [isAvailable, setIsAvailable] = useState(false); + + const streamRef = useRef(null); + const audioContextRef = useRef(null); + const workletRef = useRef(null); + const batchTimerRef = useRef | null>(null); + const audioBatchRef = useRef([]); + // Tail of the serialized flush chain. Each flush appends its IPC work to + // this promise so flushes run strictly one-after-another; awaiting the tail + // guarantees every already-started flush's chunks are enqueued native-side. + const flushChainRef = useRef>(Promise.resolve()); + const unlistenTranscriptRef = useRef(null); + const unlistenStateRef = useRef(null); + const onRecordingStartRef = useRef(onRecordingStart); + const onTranscriptTextRef = useRef(onTranscriptText); + // Native session ID — set after `start_dictation` returns. Transcript and + // state events include this ID so we can definitively ignore stale events + // from a previous session's forwarder. + const nativeSessionRef = useRef(0); + // Abort flag — set when stop/cancel is called while startRecording is still + // awaiting async setup. The start resumes and bails before activating. + const startAbortedRef = useRef(false); + + onRecordingStartRef.current = onRecordingStart; + onTranscriptTextRef.current = onTranscriptText; + + const isEnabled = !disabled && isAvailable; + + // Check availability on mount and poll until available (model may be downloading). + useEffect(() => { + let cancelled = false; + let pollTimer: ReturnType | null = null; + + function checkAvailability() { + invoke("get_dictation_status") + .then((status) => { + if (cancelled) return; + setIsAvailable(status.available); + // Stop polling once the model is ready. + if (status.available && pollTimer) { + clearInterval(pollTimer); + pollTimer = null; + } + }) + .catch(() => { + if (!cancelled) setIsAvailable(false); + }); + } + + checkAvailability(); + + // Poll periodically until available (handles background model download). + pollTimer = setInterval(() => { + if (cancelled) return; + checkAvailability(); + }, MODEL_POLL_INTERVAL_MS); + + return () => { + cancelled = true; + if (pollTimer) clearInterval(pollTimer); + }; + }, []); + + /** Flush accumulated audio batch to the native STT engine. Returns a promise + * that resolves once the IPC call completes (or immediately if nothing to flush). + * + * Flushes are serialized through `flushChainRef`: the batch is drained + * synchronously here (so each flush owns a distinct set of samples in FIFO + * order), but the actual IPC send is chained after any prior in-flight + * flush. This keeps chunks strictly ordered and lets `stopRecording`/ + * `cleanup` await the chain tail so an earlier timer-tick flush can't still + * be enqueuing chunks when `stop_dictation` fires. */ + const flushAudioBatch = useCallback((): Promise => { + const batch = audioBatchRef.current; + if (batch.length === 0) return flushChainRef.current; + + // Tag this flush with the session that owns the buffered audio. Captured + // once here so every chunk of this batch carries the same session; native + // drops chunks whose session no longer matches the active one, so a late + // flush from a just-stopped session can't leak into a newer session. + const session = nativeSessionRef.current; + + // Calculate total byte length and merge into a single buffer. Drain the + // batch synchronously so a concurrent timer tick can't grab the same + // samples — the merged buffer captured here is this flush's exclusive + // payload. + let totalSamples = 0; + for (const chunk of batch) { + totalSamples += chunk.length; + } + const merged = new Float32Array(totalSamples); + let offset = 0; + for (const chunk of batch) { + merged.set(chunk, offset); + offset += chunk.length; + } + audioBatchRef.current = []; + + // Split into chunks under the native IPC cap and send them in order. + // A single unbounded buffer can exceed the 100 KB native limit if the + // batch timer was delayed (e.g. main-thread stall), in which case native + // rejects it and the whole chunk is silently lost. + const sendChunks = async (): Promise => { + for (let start = 0; start < merged.length; start += MAX_IPC_SAMPLES) { + const slice = merged.subarray( + start, + Math.min(start + MAX_IPC_SAMPLES, merged.length), + ); + // Build the IPC payload: an 8-byte LE u64 session header followed by + // the audio bytes. Copy the slice into the header-prefixed buffer so + // the raw payload is exactly this chunk (a subarray view shares the + // parent's ArrayBuffer). + const payload = new Uint8Array(SESSION_HEADER_BYTES + slice.byteLength); + new DataView(payload.buffer).setBigUint64( + 0, + BigInt(session), + true, // little-endian + ); + payload.set( + new Uint8Array( + slice.buffer.slice( + slice.byteOffset, + slice.byteOffset + slice.byteLength, + ), + ), + SESSION_HEADER_BYTES, + ); + await invokeRawBinary("push_dictation_audio", payload).catch(() => {}); + } + }; + + // Chain this flush's IPC work after any prior in-flight flush so chunks + // stay strictly ordered and awaiting the tail drains everything. + const chained = flushChainRef.current.then(sendChunks); + flushChainRef.current = chained; + return chained; + }, []); + + const cleanup = useCallback(() => { + // Abort any in-flight startRecording so it bails after its next await + // instead of resuming and opening the mic/worklet or leaving the native + // session/listeners running after teardown. `cleanup` is the unmount + // handler (and the catch-path teardown); without this, unmounting while + // startRecording awaits `start_dictation`/`listen`/`getUserMedia`/ + // `addModule` would let the async start finish against a torn-down + // instance. A fresh startRecording clears this flag before its first + // await, so it never wrongly aborts a subsequent start. + startAbortedRef.current = true; + // Flush any remaining audio before teardown, then stop the native engine. + // Scope the stop to THIS hook instance's session. `cleanup` runs as every + // instance's unmount handler, so an unscoped stop here would let a + // non-recording composer (e.g. a thread reply composer closing) tear down + // the singleton engine owned by another composer that is actively + // recording. `nativeSessionRef.current` is 0 for an instance that never + // started a session (native IDs start at 1), so its stop can never match + // the live session and correctly no-ops. + const stoppingSession = nativeSessionRef.current; + void flushAudioBatch().then(() => { + invoke("stop_dictation", { session: stoppingSession }).catch(() => {}); + }); + // Stop batch timer. + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + // Stop mic. + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + // Disconnect audio worklet. Clear the port handler first so any PCM + // messages still queued on the main thread are dropped instead of + // appended to the (reused) audio batch. + if (workletRef.current) { + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + // Drop any audio still buffered so it can't leak into the next session. + audioBatchRef.current = []; + // Close audio context. + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + // Unlisten events. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + }, [flushAudioBatch]); + + // Cleanup on unmount. + useEffect(() => cleanup, [cleanup]); + + const startRecording = useCallback(async () => { + if (!isEnabled || isStarting || isRecording) return; + + // Clear abort flag for this new start attempt. + startAbortedRef.current = false; + // Reset any leftover audio buffer so a stale batch from a prior session + // can't be flushed into this new session/draft. + audioBatchRef.current = []; + // Reset the flush chain so this session's flushes don't chain behind a + // prior session's (already-settled) tail. + flushChainRef.current = Promise.resolve(); + + setIsStarting(true); + onRecordingStartRef.current?.(); + + try { + // 1. Start the native STT engine — returns the session ID used to tag events. + const sessionId = await invoke("start_dictation"); + nativeSessionRef.current = sessionId; + + // Bail if aborted during engine start. + if (startAbortedRef.current) { + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + // Unregister any lingering listeners from a previous session so they + // can't match the new session ID via the shared ref. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + + // 2. Listen for transcript events from the native layer. + // Each listener captures `sessionId` by value (closure) so it only + // matches events from this specific session — immune to ref mutation. + const unlistenTranscript = await listen<{ + text: string; + session: number; + }>(DICTATION_TRANSCRIPT_EVENT, (event) => { + const { text, session } = event.payload; + if (session !== sessionId) return; + if (text) { + onTranscriptTextRef.current(text); + } + }); + // Bail if stop/cancel was called while we were awaiting. + if (startAbortedRef.current) { + unlistenTranscript(); + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + unlistenTranscriptRef.current = unlistenTranscript; + + const unlistenState = await listen<{ + state: string; + session: number; + }>(DICTATION_STATE_EVENT, (event) => { + const { state, session } = event.payload; + if (session !== sessionId) return; + if (state === "stopped") { + setIsRecording(false); + setIsTranscribing(false); + // Clean up event listeners now that the session is fully done. + if (unlistenTranscriptRef.current) { + unlistenTranscriptRef.current(); + unlistenTranscriptRef.current = null; + } + if (unlistenStateRef.current) { + unlistenStateRef.current(); + unlistenStateRef.current = null; + } + } + }); + // Bail if stop/cancel was called while we were awaiting. + if (startAbortedRef.current) { + unlistenTranscript(); + unlistenState(); + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + unlistenStateRef.current = unlistenState; + + // 3. Capture mic audio. + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }); + streamRef.current = stream; + + // Bail if aborted during mic permission prompt. + if (startAbortedRef.current) { + for (const track of stream.getTracks()) track.stop(); + streamRef.current = null; + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + // 4. Set up AudioWorklet to send PCM to native layer. + const audioContext = new AudioContext({ sampleRate: 48000 }); + audioContextRef.current = audioContext; + + // Resume if the WebView created the context suspended (autoplay policy). + // A suspended context never pulls the worklet's `process()`, so no PCM + // would reach `push_dictation_audio` while the UI shows an active + // recording. Mirrors the huddle capture path (`huddle/lib/audioWorklet.ts`). + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + // Create a processor that accumulates audio frames and posts them + // to the main thread. Batching happens on the main thread side via + // a timer to reduce IPC overhead. + const processorCode = ` + class DictationProcessor extends AudioWorkletProcessor { + process(inputs) { + const input = inputs[0]; + if (input && input[0] && input[0].length > 0) { + this.port.postMessage(input[0].buffer); + } + return true; + } + } + registerProcessor('dictation-processor', DictationProcessor); + `; + const blob = new Blob([processorCode], { + type: "application/javascript", + }); + const blobUrl = URL.createObjectURL(blob); + try { + await audioContext.audioWorklet.addModule(blobUrl); + } finally { + URL.revokeObjectURL(blobUrl); + } + + // Bail if stop/cancel was called while the worklet module was loading. + // Without this the worklet/flush timer would start and `isRecording` + // would be set true, leaving dictation running after it was stopped. + if (startAbortedRef.current) { + for (const track of stream.getTracks()) track.stop(); + streamRef.current = null; + void audioContext.close(); + audioContextRef.current = null; + invoke("stop_dictation", { session: sessionId }).catch(() => {}); + return; + } + + const source = audioContext.createMediaStreamSource(stream); + const worklet = new AudioWorkletNode(audioContext, "dictation-processor"); + workletRef.current = worklet; + + // Accumulate audio frames in a batch array; a timer flushes to native. + worklet.port.onmessage = (event: MessageEvent) => { + audioBatchRef.current.push(new Float32Array(event.data)); + }; + + // Start the batch flush timer (~10 IPC calls/s instead of ~375). + batchTimerRef.current = setInterval(flushAudioBatch, AUDIO_BATCH_MS); + + source.connect(worklet); + // Don't connect to destination — we only capture, not play back. + worklet.connect(audioContext.createGain()); // keep worklet alive without audible output + + setIsRecording(true); + setIsTranscribing(true); + } catch (error) { + // Tear down and stop the native engine if it was started but a later + // step failed (e.g. mic permission denied, AudioWorklet setup error). + // `cleanup()` performs the session-scoped stop, so it only tears down + // this instance's own session — never another composer's. + cleanup(); + setIsRecording(false); + setIsTranscribing(false); + + const message = + error instanceof Error ? error.message : "Local dictation failed"; + if (/not allowed|denied|permission/i.test(message)) { + toast.error("Microphone access denied", { + description: + "Allow microphone access in System Settings to use dictation.", + }); + } else if (/not found|no audio/i.test(message)) { + toast.error("No microphone found", { + description: "Connect a microphone and try again.", + }); + } else if (/model not ready/i.test(message)) { + toast.error("Voice model downloading", { + description: + "The speech model is still downloading. Try again shortly.", + }); + } else { + toast.error("Dictation failed", { description: message }); + } + } finally { + setIsStarting(false); + } + }, [cleanup, flushAudioBatch, isEnabled, isRecording, isStarting]); + + const stopRecording = useCallback(() => { + // Signal any in-flight startRecording to bail after its next await. + startAbortedRef.current = true; + // Stop batch timer immediately. + if (batchTimerRef.current) { + clearInterval(batchTimerRef.current); + batchTimerRef.current = null; + } + // Stop mic and audio pipeline immediately so the user gets visual feedback. + if (streamRef.current) { + for (const track of streamRef.current.getTracks()) { + track.stop(); + } + streamRef.current = null; + } + if (workletRef.current) { + // Clear the port handler so PCM messages still queued on the main + // thread are dropped rather than appended to the batch after the + // final flush below (and leaking into the next session). + workletRef.current.port.onmessage = null; + workletRef.current.disconnect(); + workletRef.current = null; + } + if (audioContextRef.current) { + void audioContextRef.current.close(); + audioContextRef.current = null; + } + setIsRecording(false); + // Flush remaining audio and THEN stop the native engine, ensuring the + // final batch arrives before the engine shuts down and flushes its buffer. + // isTranscribing stays true — cleared when `dictation-state: stopped` arrives. + // Scope the stop to THIS session: if the user restarts during the flush + // window, a new session may already be stored by the time this resolves — + // passing the session keeps this stop from tearing down the new engine. + const stoppingSession = nativeSessionRef.current; + void flushAudioBatch().then(() => { + invoke("stop_dictation", { session: stoppingSession }).catch(() => {}); + }); + }, [flushAudioBatch]); + + const cancelRecording = useCallback(() => { + // Signal any in-flight startRecording to bail after its next await. + startAbortedRef.current = true; + // `cleanup()` performs the session-scoped native stop, so cancelling a + // composer that isn't the active recorder can't tear down another + // composer's session. + cleanup(); + setIsRecording(false); + setIsTranscribing(false); + }, [cleanup]); + + const toggleRecording = useCallback(() => { + if (isRecording || isStarting) { + stopRecording(); + return; + } + void startRecording(); + }, [isRecording, isStarting, startRecording, stopRecording]); + + return { + isEnabled, + isRecording, + isStarting, + isTranscribing, + startRecording, + stopRecording, + cancelRecording, + toggleRecording, + }; +} diff --git a/desktop/src/features/dictation/index.ts b/desktop/src/features/dictation/index.ts new file mode 100644 index 0000000000..1578aaf655 --- /dev/null +++ b/desktop/src/features/dictation/index.ts @@ -0,0 +1,3 @@ +export { useComposerDictation } from "./hooks/useComposerDictation"; +export { useDictation } from "./hooks/useDictation"; +export { DictationButton } from "./ui/DictationButton"; diff --git a/desktop/src/features/dictation/lib/activeComposer.ts b/desktop/src/features/dictation/lib/activeComposer.ts new file mode 100644 index 0000000000..1fb1b3f104 --- /dev/null +++ b/desktop/src/features/dictation/lib/activeComposer.ts @@ -0,0 +1,23 @@ +/** + * Tracks which composer instance should handle global dictation shortcuts. + * + * When multiple composers are mounted (e.g. channel + thread), only the + * most recently focused one should respond to ⌘D. Each composer registers + * on focus and the global shortcut only dispatches to the active instance. + */ + +let activeInstanceId: string | null = null; + +export function setActiveDictationComposer(id: string): void { + activeInstanceId = id; +} + +export function clearActiveDictationComposer(id: string): void { + if (activeInstanceId === id) { + activeInstanceId = null; + } +} + +export function isActiveDictationComposer(id: string): boolean { + return activeInstanceId === id; +} diff --git a/desktop/src/features/dictation/lib/voiceInput.test.mjs b/desktop/src/features/dictation/lib/voiceInput.test.mjs new file mode 100644 index 0000000000..151104cbd8 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.test.mjs @@ -0,0 +1,150 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + DEFAULT_AUTO_SUBMIT_PHRASE, + getAutoSubmitMatch, + parseAutoSubmitPhrases, + replaceTrailingTranscribedText, +} from "./voiceInput.ts"; + +// ── parseAutoSubmitPhrases ────────────────────────────────────────────────── + +test("parseAutoSubmitPhrases_returnsEmptyForNullish", () => { + assert.deepEqual(parseAutoSubmitPhrases(null), []); + assert.deepEqual(parseAutoSubmitPhrases(undefined), []); + assert.deepEqual(parseAutoSubmitPhrases(""), []); +}); + +test("parseAutoSubmitPhrases_splitsNormalizesAndDedupes", () => { + assert.deepEqual(parseAutoSubmitPhrases("Submit, send it, submit, "), [ + "submit", + "send it", + ]); +}); + +test("parseAutoSubmitPhrases_stripsTrailingPunctuation", () => { + assert.deepEqual(parseAutoSubmitPhrases("submit!"), ["submit"]); +}); + +// ── replaceTrailingTranscribedText ────────────────────────────────────────── + +test("replaceTrailingTranscribedText_appendsWhenNoPrevious", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", "world"), + "Hello world", + ); +}); + +test("replaceTrailingTranscribedText_appendsToEmptyBase", () => { + assert.equal(replaceTrailingTranscribedText("", "", "hello"), "hello"); +}); + +test("replaceTrailingTranscribedText_replacesTrailingInterim", () => { + // Interim "hello wor" is refined to "hello world". + assert.equal( + replaceTrailingTranscribedText("hello wor", "hello wor", "hello world"), + "hello world", + ); +}); + +test("replaceTrailingTranscribedText_preservesTextTypedBeforeDictation", () => { + // User typed "Note: " then dictated; the manual prefix must survive. + assert.equal( + replaceTrailingTranscribedText("Note: hi", "hi", "hi there"), + "Note: hi there", + ); +}); + +test("replaceTrailingTranscribedText_appendsWhenPreviousNoLongerMatches", () => { + // If the previous transcript isn't the trailing text anymore, append. + assert.equal( + replaceTrailingTranscribedText("edited text", "old", "new"), + "edited text new", + ); +}); + +test("replaceTrailingTranscribedText_noDoubleSpaceBeforePunctuation", () => { + assert.equal( + replaceTrailingTranscribedText("Hello", "", ", world"), + "Hello, world", + ); +}); + +// ── getAutoSubmitMatch ────────────────────────────────────────────────────── + +test("getAutoSubmitMatch_returnsNullWhenPhraseAbsent", () => { + assert.equal( + getAutoSubmitMatch("hello there", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_returnsNullWhenPhrasesEmpty", () => { + // DEFAULT_AUTO_SUBMIT_PHRASE is empty (auto-submit disabled by default). + assert.equal( + getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases(DEFAULT_AUTO_SUBMIT_PHRASE), + ), + null, + ); +}); + +test("getAutoSubmitMatch_matchesTrailingPhraseAndStripsIt", () => { + const match = getAutoSubmitMatch( + "send this message submit", + parseAutoSubmitPhrases("submit"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit"); + assert.equal(match.textWithoutPhrase, "send this message"); +}); + +test("getAutoSubmitMatch_ignoresPhraseMidSentence", () => { + // "submit" is not at the end, so it must not auto-send. + assert.equal( + getAutoSubmitMatch( + "submit the form later", + parseAutoSubmitPhrases("submit"), + ), + null, + ); +}); + +test("getAutoSubmitMatch_requiresWordBoundaryBeforePhrase", () => { + // "resubmit" ends with "submit" but is not a standalone word → no match. + assert.equal( + getAutoSubmitMatch("resubmit", parseAutoSubmitPhrases("submit")), + null, + ); +}); + +test("getAutoSubmitMatch_toleratesTrailingPunctuation", () => { + const match = getAutoSubmitMatch( + "ship it submit.", + parseAutoSubmitPhrases("submit"), + ); + assert.ok(match); + assert.equal(match.textWithoutPhrase, "ship it"); +}); + +test("getAutoSubmitMatch_matchesMultiWordPhrase", () => { + const match = getAutoSubmitMatch( + "please do this send it", + parseAutoSubmitPhrases("send it"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "send it"); + assert.equal(match.textWithoutPhrase, "please do this"); +}); + +test("getAutoSubmitMatch_prefersLongestPhrase", () => { + const match = getAutoSubmitMatch( + "text please submit now", + parseAutoSubmitPhrases("submit now, now"), + ); + assert.ok(match); + assert.equal(match.matchedPhrase, "submit now"); + assert.equal(match.textWithoutPhrase, "text please"); +}); diff --git a/desktop/src/features/dictation/lib/voiceInput.ts b/desktop/src/features/dictation/lib/voiceInput.ts new file mode 100644 index 0000000000..1c287cef72 --- /dev/null +++ b/desktop/src/features/dictation/lib/voiceInput.ts @@ -0,0 +1,114 @@ +/** + * Default auto-submit phrase. Empty string disables auto-submit — the user + * must manually press Enter/Send after dictation. The infrastructure for + * configurable phrases is in place (parseAutoSubmitPhrases, getAutoSubmitMatch) + * and can be wired to a user setting when we're ready to ship auto-submit. + */ +export const DEFAULT_AUTO_SUBMIT_PHRASE = ""; + +const TRAILING_PUNCTUATION_REGEX = /[\s"'`.,!?;:)\]}]+$/u; + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function normalizePhrase(value: string): string { + return value + .toLowerCase() + .replace(/\s+/g, " ") + .trim() + .replace(TRAILING_PUNCTUATION_REGEX, "") + .trim(); +} + +export function parseAutoSubmitPhrases( + rawValue: string | null | undefined, +): string[] { + if (!rawValue) return []; + return Array.from( + new Set( + rawValue + .split(",") + .map((value) => normalizePhrase(value)) + .filter(Boolean), + ), + ); +} + +function appendTranscribedText(baseText: string, fragment: string): string { + const normalizedFragment = fragment.replace(/\s+/g, " ").trim(); + if (!normalizedFragment) return baseText; + if (!baseText.trim()) return normalizedFragment; + if (/[\s([{/-]$/.test(baseText) || /^[,.;!?)]/.test(normalizedFragment)) { + return `${baseText}${normalizedFragment}`; + } + return `${baseText} ${normalizedFragment}`; +} + +export function replaceTrailingTranscribedText( + fullText: string, + previousTranscribedText: string, + nextTranscribedText: string, +): string { + if (!previousTranscribedText) { + return appendTranscribedText(fullText, nextTranscribedText); + } + + if (fullText.endsWith(previousTranscribedText)) { + return appendTranscribedText( + fullText.slice(0, -previousTranscribedText.length), + nextTranscribedText, + ); + } + + const trimmedPreviousText = previousTranscribedText.trim(); + if (trimmedPreviousText && fullText.endsWith(trimmedPreviousText)) { + return appendTranscribedText( + fullText.slice(0, -trimmedPreviousText.length), + nextTranscribedText, + ); + } + + return appendTranscribedText(fullText, nextTranscribedText); +} + +export function getAutoSubmitMatch( + transcribedText: string, + autoSubmitPhrases: string[], +): { matchedPhrase: string; textWithoutPhrase: string } | null { + const normalizedTranscribedText = normalizePhrase(transcribedText); + if (!normalizedTranscribedText) return null; + + const sortedPhrases = [...autoSubmitPhrases].sort( + (left, right) => right.length - left.length, + ); + + for (const phrase of sortedPhrases) { + if (!normalizedTranscribedText.endsWith(phrase)) continue; + + const phraseStartIndex = normalizedTranscribedText.length - phrase.length; + if ( + phraseStartIndex > 0 && + normalizedTranscribedText[phraseStartIndex - 1] !== " " + ) { + continue; + } + + const trimmedText = transcribedText.replace(TRAILING_PUNCTUATION_REGEX, ""); + const phraseWords = phrase.split(" ").filter(Boolean).map(escapeRegExp); + const phrasePattern = new RegExp( + `(^|\\s)(${phraseWords.join("\\s+")})\\s*$`, + "iu", + ); + const rawMatch = trimmedText.match(phrasePattern); + const phraseStartOffset = + rawMatch && rawMatch.index !== undefined + ? rawMatch.index + (rawMatch[1]?.length ?? 0) + : trimmedText.length - phrase.length; + const textWithoutPhrase = trimmedText.slice(0, phraseStartOffset).trimEnd(); + + return { matchedPhrase: phrase, textWithoutPhrase }; + } + + return null; +} diff --git a/desktop/src/features/dictation/ui/DictationButton.tsx b/desktop/src/features/dictation/ui/DictationButton.tsx new file mode 100644 index 0000000000..f284957e17 --- /dev/null +++ b/desktop/src/features/dictation/ui/DictationButton.tsx @@ -0,0 +1,73 @@ +import { Mic, Square } from "lucide-react"; +import { Button } from "@/shared/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/shared/ui/tooltip"; +import { cn } from "@/shared/lib/cn"; +import { isMacPlatform } from "@/shared/lib/platform"; + +interface DictationState { + isEnabled: boolean; + isRecording: boolean; + isStarting: boolean; + isTranscribing: boolean; + toggleRecording: () => void; +} + +interface DictationButtonProps { + dictation: DictationState; + disabled?: boolean; +} + +export function DictationButton({ + dictation, + disabled = false, +}: DictationButtonProps) { + if (!dictation.isEnabled) return null; + + const shortcutHint = isMacPlatform() ? "⌘D" : "Ctrl+D"; + + const tooltipText = dictation.isRecording + ? "Stop recording" + : dictation.isTranscribing + ? "Transcribing…" + : null; + + return ( + + + + + + {tooltipText ?? ( + + Voice Dictation + + {shortcutHint} + + + )} + + + ); +} diff --git a/desktop/src/features/messages/ui/MessageComposer.tsx b/desktop/src/features/messages/ui/MessageComposer.tsx index 51a599fbc9..2ce6533b5b 100644 --- a/desktop/src/features/messages/ui/MessageComposer.tsx +++ b/desktop/src/features/messages/ui/MessageComposer.tsx @@ -54,6 +54,7 @@ import { NonMemberMentionDialog } from "./NonMemberMentionDialog"; import { useMentionSendFlow } from "./useMentionSendFlow"; import { useComposerContentState } from "./useComposerContentState"; import { useDraftPersistLifecycle } from "./useDraftPersistSnapshot"; +import { DictationButton, useComposerDictation } from "@/features/dictation"; type MessageComposerProps = { channelId?: string | null; @@ -257,7 +258,6 @@ function MessageComposerImpl({ onEditSaveRef.current = onEditSave; onEditLastOwnMessageRef.current = onEditLastOwnMessage; editTargetRef.current = editTarget; - const isAutocompleteOpenRef = React.useRef(false); isAutocompleteOpenRef.current = mentions.isMentionOpen || @@ -265,8 +265,22 @@ function MessageComposerImpl({ emojiAutocomplete.isEmojiAutocompleteOpen; const submitMessageRef = React.useRef<() => void>(() => {}); + const setEditorContentRef = React.useRef<(text: string) => void>(() => {}); + const stopDictationRef = React.useRef<() => void>(() => {}); const composerScrollRef = React.useRef(null); - + const dictation = useComposerDictation({ + syncContentRef: syncContentRefFromEditorRef, + disabled, + disabledRef, + isSendingRef, + isUploadingRef, + setComposerContent, + setEditorContentRef, + submitMessageRef, + draftKey: effectiveDraftKey, + composerRef: composerScrollRef, + }); + stopDictationRef.current = dictation.cancelRecording; // Set after `useLinkEditor` exists below; the editor's link-click handler // delegates through this ref to break the hook ordering cycle (the editor // needs `onEditLink`, but the link editor needs the editor's `richText`). @@ -323,6 +337,7 @@ function MessageComposerImpl({ }, }); + setEditorContentRef.current = richText.setContent; const linkEditor = useLinkEditor(richText); syncContentRefFromEditorRef.current = () => { const markdown = richText.getMarkdown(); @@ -541,6 +556,7 @@ function MessageComposerImpl({ // Edit mode if (editTargetRef.current && onEditSaveRef.current) { if (isSendingRef.current || isUploadingRef.current) return; + stopDictationRef.current(); // cancel dictation before edit save const currentPendingImeta = media.pendingImetaRef.current; const hasMedia = currentPendingImeta.length > 0; // Empty text + zero attachments is a no-op (don't let edit become an @@ -557,11 +573,8 @@ function MessageComposerImpl({ spoileredAttachmentUrls, ); - // NIP-30: attach `["emoji", shortcode, url]` tags for custom emoji in the - // edited body, exactly like the send path. Without this an edited message - // ships with no emoji tags, so the receiver can't resolve a `:shortcode:` - // and renders the literal text. `?? []` preserves edit semantics (a - // defined-but-empty media set means "wipe attachments"). + // NIP-30 emoji tags for the edited body (mirrors the send path). + // `?? []` preserves edit semantics (empty = "wipe attachments"). const outgoingTags = mergeOutgoingTags( mediaTags, @@ -603,6 +616,7 @@ function MessageComposerImpl({ ) { return; } + stopDictationRef.current(); // cancel dictation; send proceeds with current content const capturedThreadContext = onCaptureSendContext?.() ?? null; // If a thread-reply composer reported no reply target at submit time, @@ -696,10 +710,8 @@ function MessageComposerImpl({ ); // ── Keyboard handling ─────────────────────────────────────────────── - // Tiptap handles formatting shortcuts (⌘B, ⌘I, etc.) natively. - // Plain Enter → submit is now handled inside the Tiptap `submitOnEnter` - // extension (fires before ProseMirror's splitBlock). This wrapper only - // handles autocomplete arrow/enter keys and Escape for edit mode. + // Autocomplete arrow/enter keys and Escape for edit mode only; plain + // Enter → submit is in the Tiptap `submitOnEnter` extension. const handleEditorKeyDown = React.useCallback( (event: React.KeyboardEvent) => { // Let autocomplete handle keys first @@ -994,7 +1006,12 @@ function MessageComposerImpl({ + + {toolbarExtraActions} + + } formattingDisabled={disabled} isEmojiPickerOpen={isEmojiPickerOpen} isFormattingOpen={isFormattingOpen} @@ -1028,5 +1045,4 @@ function MessageComposerImpl({ ); } - export const MessageComposer = React.memo(MessageComposerImpl); diff --git a/desktop/src/shared/lib/keyboard-shortcuts.ts b/desktop/src/shared/lib/keyboard-shortcuts.ts index 84c474afb2..5f6e4b758b 100644 --- a/desktop/src/shared/lib/keyboard-shortcuts.ts +++ b/desktop/src/shared/lib/keyboard-shortcuts.ts @@ -181,6 +181,14 @@ export const KEYBOARD_SHORTCUTS: KeyboardShortcut[] = [ keysWindows: "Ctrl+Space", category: "Messages", }, + { + id: "voice-dictation", + label: "Voice Dictation", + description: "Hold to dictate, release to stop (push-to-talk)", + keys: "⌘D", + keysWindows: "Ctrl+D", + category: "Messages", + }, // Formatting {