From f5f11b7cdfe21cc41b50cd8edca156c7d1455d6c Mon Sep 17 00:00:00 2001 From: Nolan Darilek Date: Tue, 22 Sep 2020 14:51:59 -0500 Subject: [PATCH 1/4] Switch to using MediaPlaybackItem as WinRT utterance ID. --- src/backends/winrt.rs | 13 ++----------- src/lib.rs | 7 +++++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/backends/winrt.rs b/src/backends/winrt.rs index 32fb219..f558c91 100644 --- a/src/backends/winrt.rs +++ b/src/backends/winrt.rs @@ -1,7 +1,4 @@ #[cfg(windows)] -use std::sync::Mutex; - -use lazy_static::lazy_static; use log::{info, trace}; use tts_winrt_bindings::windows::media::core::MediaSource; @@ -24,10 +21,6 @@ pub struct WinRT { playback_list: MediaPlaybackList, } -lazy_static! { - static ref NEXT_UTTERANCE_ID: Mutex = Mutex::new(0); -} - impl WinRT { pub fn new() -> std::result::Result { info!("Initializing WinRT backend"); @@ -83,13 +76,11 @@ impl Backend for WinRT { self.reinit_player()?; } } - self.playback_list.items()?.append(item)?; + self.playback_list.items()?.append(&item)?; if !self.is_speaking()? { self.player.play()?; } - let mut utterance_id = NEXT_UTTERANCE_ID.lock().unwrap(); - *utterance_id += 1; - Ok(Some(UtteranceId::WinRT(*utterance_id))) + Ok(Some(UtteranceId::WinRT(item))) } fn stop(&mut self) -> std::result::Result<(), Error> { diff --git a/src/lib.rs b/src/lib.rs index 2d494cf..fc0558a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,6 +23,9 @@ use libc::c_char; use objc::{class, msg_send, sel, sel_impl}; use thiserror::Error; +#[cfg(windows)] +use tts_winrt_bindings::windows::media::playback::MediaPlaybackItem; + mod backends; pub enum Backends { @@ -40,14 +43,14 @@ pub enum Backends { AvFoundation, } -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum UtteranceId { #[cfg(target_os = "linux")] SpeechDispatcher(i32), #[cfg(target_arch = "wasm32")] Web(u64), #[cfg(windows)] - WinRT(u64), + WinRT(MediaPlaybackItem), } pub struct Features { From 61522610cd851850f0f152224e4bf23255005162 Mon Sep 17 00:00:00 2001 From: Nolan Darilek Date: Wed, 23 Sep 2020 10:12:51 -0500 Subject: [PATCH 2/4] Implement utterance begin/end callback framework, and set up for Speech-Dispatcher. --- examples/hello_world.rs | 12 +++++ src/backends/mod.rs | 2 +- src/backends/speech_dispatcher.rs | 32 ++++++++--- src/lib.rs | 90 ++++++++++++++++++++++++++++++- 4 files changed, 127 insertions(+), 9 deletions(-) diff --git a/examples/hello_world.rs b/examples/hello_world.rs index e95c53c..9b1954b 100644 --- a/examples/hello_world.rs +++ b/examples/hello_world.rs @@ -12,6 +12,18 @@ use tts::*; fn main() -> Result<(), Error> { env_logger::init(); let mut tts = TTS::default()?; + let Features { + utterance_callbacks, + .. + } = tts.supported_features(); + if utterance_callbacks { + tts.on_utterance_begin(Some(|utterance| { + println!("Started speaking {:?}", utterance) + }))?; + tts.on_utterance_end(Some(|utterance| { + println!("Finished speaking {:?}", utterance) + }))?; + } tts.speak("Hello, world.", false)?; let Features { rate, .. } = tts.supported_features(); if rate { diff --git a/src/backends/mod.rs b/src/backends/mod.rs index c999faf..a9f094f 100644 --- a/src/backends/mod.rs +++ b/src/backends/mod.rs @@ -17,7 +17,7 @@ mod appkit; mod av_foundation; #[cfg(target_os = "linux")] -pub use self::speech_dispatcher::*; +pub(crate) use self::speech_dispatcher::*; #[cfg(windows)] pub use self::tolk::*; diff --git a/src/backends/speech_dispatcher.rs b/src/backends/speech_dispatcher.rs index 5717c61..2b9d6b5 100644 --- a/src/backends/speech_dispatcher.rs +++ b/src/backends/speech_dispatcher.rs @@ -1,14 +1,15 @@ #[cfg(target_os = "linux")] use std::collections::HashMap; +use std::convert::TryInto; use std::sync::Mutex; use lazy_static::*; use log::{info, trace}; use speech_dispatcher::*; -use crate::{Backend, Error, Features, UtteranceId}; +use crate::{Backend, BackendId, Error, Features, UtteranceId, CALLBACKS}; -pub struct SpeechDispatcher(Connection); +pub(crate) struct SpeechDispatcher(Connection); lazy_static! { static ref SPEAKING: Mutex> = { @@ -18,19 +19,33 @@ lazy_static! { } impl SpeechDispatcher { - pub fn new() -> Self { + pub(crate) fn new() -> Self { info!("Initializing SpeechDispatcher backend"); let connection = speech_dispatcher::Connection::open("tts", "tts", "tts", Mode::Threaded); let sd = SpeechDispatcher(connection); let mut speaking = SPEAKING.lock().unwrap(); speaking.insert(sd.0.client_id(), false); - sd.0.on_begin(Some(|_msg_id, client_id| { + sd.0.on_begin(Some(|msg_id, client_id| { let mut speaking = SPEAKING.lock().unwrap(); speaking.insert(client_id, true); + let callbacks = CALLBACKS.lock().unwrap(); + let backend_id = BackendId::SpeechDispatcher(client_id); + let cb = callbacks.get(&backend_id).unwrap(); + let utterance_id = UtteranceId::SpeechDispatcher(msg_id); + if let Some(f) = cb.utterance_begin { + f(utterance_id); + } })); - sd.0.on_end(Some(|_msg_id, client_id| { + sd.0.on_end(Some(|msg_id, client_id| { let mut speaking = SPEAKING.lock().unwrap(); speaking.insert(client_id, false); + let callbacks = CALLBACKS.lock().unwrap(); + let backend_id = BackendId::SpeechDispatcher(client_id); + let cb = callbacks.get(&backend_id).unwrap(); + let utterance_id = UtteranceId::SpeechDispatcher(msg_id); + if let Some(f) = cb.utterance_end { + f(utterance_id); + } })); sd.0.on_cancel(Some(|_msg_id, client_id| { let mut speaking = SPEAKING.lock().unwrap(); @@ -49,6 +64,10 @@ impl SpeechDispatcher { } impl Backend for SpeechDispatcher { + fn id(&self) -> Option { + Some(BackendId::SpeechDispatcher(self.0.client_id())) + } + fn supported_features(&self) -> Features { Features { stop: true, @@ -56,6 +75,7 @@ impl Backend for SpeechDispatcher { pitch: true, volume: true, is_speaking: true, + utterance_callbacks: true, } } @@ -73,7 +93,7 @@ impl Backend for SpeechDispatcher { self.0.set_punctuation(Punctuation::None); } if let Some(id) = id { - Ok(Some(UtteranceId::SpeechDispatcher(id))) + Ok(Some(UtteranceId::SpeechDispatcher(id.try_into().unwrap()))) } else { Err(Error::NoneError) } diff --git a/src/lib.rs b/src/lib.rs index fc0558a..c98a625 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,11 +12,14 @@ */ use std::boxed::Box; +use std::collections::HashMap; #[cfg(target_os = "macos")] use std::ffi::CStr; +use std::sync::Mutex; #[cfg(target_os = "macos")] use cocoa_foundation::base::id; +use lazy_static::lazy_static; #[cfg(target_os = "macos")] use libc::c_char; #[cfg(target_os = "macos")] @@ -43,10 +46,20 @@ pub enum Backends { AvFoundation, } +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub enum BackendId { + #[cfg(target_os = "linux")] + SpeechDispatcher(u64), + #[cfg(target_arch = "wasm32")] + Web(u64), + #[cfg(windows)] + WinRT(MediaPlaybackItem), +} + #[derive(Clone, Debug, PartialEq)] pub enum UtteranceId { #[cfg(target_os = "linux")] - SpeechDispatcher(i32), + SpeechDispatcher(u64), #[cfg(target_arch = "wasm32")] Web(u64), #[cfg(windows)] @@ -59,6 +72,7 @@ pub struct Features { pub pitch: bool, pub volume: bool, pub is_speaking: bool, + pub utterance_callbacks: bool, } impl Default for Features { @@ -69,6 +83,7 @@ impl Default for Features { pitch: false, volume: false, is_speaking: false, + utterance_callbacks: false, } } } @@ -92,6 +107,7 @@ pub enum Error { } pub trait Backend { + fn id(&self) -> Option; fn supported_features(&self) -> Features; fn speak(&mut self, text: &str, interrupt: bool) -> Result, Error>; fn stop(&mut self) -> Result<(), Error>; @@ -113,6 +129,19 @@ pub trait Backend { fn is_speaking(&self) -> Result; } +#[derive(Default)] +struct Callbacks { + utterance_begin: Option, + utterance_end: Option, +} + +lazy_static! { + static ref CALLBACKS: Mutex> = { + let m: HashMap = HashMap::new(); + Mutex::new(m) + }; +} + pub struct TTS(Box); unsafe impl std::marker::Send for TTS {} @@ -124,7 +153,7 @@ impl TTS { * Create a new `TTS` instance with the specified backend. */ pub fn new(backend: Backends) -> Result { - match backend { + let backend = match backend { #[cfg(target_os = "linux")] Backends::SpeechDispatcher => Ok(TTS(Box::new(backends::SpeechDispatcher::new()))), #[cfg(target_arch = "wasm32")] @@ -150,6 +179,16 @@ impl TTS { Backends::AppKit => Ok(TTS(Box::new(backends::AppKit::new()))), #[cfg(any(target_os = "macos", target_os = "ios"))] Backends::AvFoundation => Ok(TTS(Box::new(backends::AvFoundation::new()))), + }; + let mut callbacks = CALLBACKS.lock().unwrap(); + if backend.is_ok() { + let backend = backend.unwrap(); + if let Some(id) = backend.0.id() { + callbacks.insert(id, Callbacks::default()); + } + Ok(backend) + } else { + backend } } @@ -387,4 +426,51 @@ impl TTS { Err(Error::UnsupportedFeature) } } + + /** + * Called when this speech synthesizer begins speaking an utterance. + */ + pub fn on_utterance_begin(&self, callback: Option) -> Result<(), Error> { + let Features { + utterance_callbacks, + .. + } = self.supported_features(); + if utterance_callbacks { + let mut callbacks = CALLBACKS.lock().unwrap(); + let id = self.0.id().unwrap(); + let mut callbacks = callbacks.get_mut(&id).unwrap(); + callbacks.utterance_begin = callback; + Ok(()) + } else { + Err(Error::UnsupportedFeature) + } + } + + /** + * Called when this speech synthesizer finishes speaking an utterance. + */ + pub fn on_utterance_end(&self, callback: Option) -> Result<(), Error> { + let Features { + utterance_callbacks, + .. + } = self.supported_features(); + if utterance_callbacks { + let mut callbacks = CALLBACKS.lock().unwrap(); + let id = self.0.id().unwrap(); + let mut callbacks = callbacks.get_mut(&id).unwrap(); + callbacks.utterance_end = callback; + Ok(()) + } else { + Err(Error::UnsupportedFeature) + } + } +} + +impl Drop for TTS { + fn drop(&mut self) { + if let Some(id) = self.0.id() { + let mut callbacks = CALLBACKS.lock().unwrap(); + callbacks.remove(&id); + } + } } From 6788277a4dadcaddc3b9f72502a5f90fafae82cc Mon Sep 17 00:00:00 2001 From: Nolan Darilek Date: Wed, 23 Sep 2020 10:31:21 -0500 Subject: [PATCH 3/4] Implement framework for utterance callbacks in Windows backends, though they aren't currently called. --- src/backends/tolk.rs | 6 +++++- src/backends/winrt.rs | 23 ++++++++++++++++++++--- src/lib.rs | 4 ++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/backends/tolk.rs b/src/backends/tolk.rs index 28c3f42..0be20d1 100644 --- a/src/backends/tolk.rs +++ b/src/backends/tolk.rs @@ -2,7 +2,7 @@ use log::{info, trace}; use tolk::Tolk as TolkPtr; -use crate::{Backend, Error, Features, UtteranceId}; +use crate::{Backend, BackendId, Error, Features, UtteranceId}; pub struct Tolk(TolkPtr); @@ -19,6 +19,10 @@ impl Tolk { } impl Backend for Tolk { + fn id(&self) -> Option { + None + } + fn supported_features(&self) -> Features { Features { stop: true, diff --git a/src/backends/winrt.rs b/src/backends/winrt.rs index f558c91..c72040c 100644 --- a/src/backends/winrt.rs +++ b/src/backends/winrt.rs @@ -1,4 +1,7 @@ #[cfg(windows)] +use std::sync::Mutex; + +use lazy_static::lazy_static; use log::{info, trace}; use tts_winrt_bindings::windows::media::core::MediaSource; @@ -7,7 +10,7 @@ use tts_winrt_bindings::windows::media::playback::{ }; use tts_winrt_bindings::windows::media::speech_synthesis::SpeechSynthesizer; -use crate::{Backend, Error, Features, UtteranceId}; +use crate::{Backend, BackendId, Error, Features, UtteranceId}; impl From for Error { fn from(e: winrt::Error) -> Self { @@ -16,11 +19,16 @@ impl From for Error { } pub struct WinRT { + id: BackendId, synth: SpeechSynthesizer, player: MediaPlayer, playback_list: MediaPlaybackList, } +lazy_static! { + static ref NEXT_BACKEND_ID: Mutex = Mutex::new(0); +} + impl WinRT { pub fn new() -> std::result::Result { info!("Initializing WinRT backend"); @@ -28,11 +36,15 @@ impl WinRT { let player = MediaPlayer::new()?; player.set_auto_play(true)?; player.set_source(&playback_list)?; - Ok(Self { + let mut backend_id = NEXT_BACKEND_ID.lock().unwrap(); + let rv = Ok(Self { + id: BackendId::WinRT(*backend_id), synth: SpeechSynthesizer::new()?, player: player, playback_list: playback_list, - }) + }); + *backend_id += 1; + rv } fn reinit_player(&mut self) -> std::result::Result<(), Error> { @@ -45,6 +57,10 @@ impl WinRT { } impl Backend for WinRT { + fn id(&self) -> Option { + Some(self.id) + } + fn supported_features(&self) -> Features { Features { stop: true, @@ -52,6 +68,7 @@ impl Backend for WinRT { pitch: true, volume: true, is_speaking: true, + ..Default::default() } } diff --git a/src/lib.rs b/src/lib.rs index c98a625..e4c6d8c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,14 +46,14 @@ pub enum Backends { AvFoundation, } -#[derive(Clone, Debug, Eq, Hash, PartialEq)] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub enum BackendId { #[cfg(target_os = "linux")] SpeechDispatcher(u64), #[cfg(target_arch = "wasm32")] Web(u64), #[cfg(windows)] - WinRT(MediaPlaybackItem), + WinRT(u64), } #[derive(Clone, Debug, PartialEq)] From c5524113ff286682e0926bd3962190b826d603ce Mon Sep 17 00:00:00 2001 From: Nolan Darilek Date: Wed, 23 Sep 2020 10:33:30 -0500 Subject: [PATCH 4/4] Document the fact that we only need an NSRunLoop in the example because there isn't one already. --- examples/hello_world.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/hello_world.rs b/examples/hello_world.rs index 9b1954b..4d7accd 100644 --- a/examples/hello_world.rs +++ b/examples/hello_world.rs @@ -61,6 +61,8 @@ fn main() -> Result<(), Error> { } tts.speak("Goodbye.", false)?; let mut _input = String::new(); + // The below is only needed to make the example run on MacOS because there is no NSRunLoop in this context. + // It shouldn't be needed in an app or game that almost certainly has one already. #[cfg(target_os = "macos")] { let run_loop: id = unsafe { NSRunLoop::currentRunLoop() };