synapto_interface/
lib.rs

1pub mod cognitive_output_audio;
2pub mod cognitive_output_text;
3pub mod peer_input_audio;
4pub mod peer_input_text;
5pub mod speech_to_text;
6pub mod storage;
7
8/// Instrumented synchronization primitives and re-exports of `tokio::sync`.
9pub mod sync;
10/// Core data types used across the interface and core engine.
11pub mod types;
12
13pub mod llm;
14
15use async_trait::async_trait;
16
17use crate::cognitive_output_audio::types::CognitiveOutputAudio;
18use crate::peer_input_audio::types::PeerInputAudio;
19use crate::speech_to_text::types::{
20    InputVoiceAudio, SpeakerSegment, SpeechDetected, SpeechTranscript,
21};
22use crate::sync::{broadcast, mpsc, watch};
23use crate::types::CognitiveOutputSpeech;
24
25#[async_trait]
26pub trait AudioInputPlugin: Plugin + Send + Sync {
27    async fn start(&self, tx: mpsc::Sender<PeerInputAudio>) -> Result<(), String>;
28}
29
30#[async_trait]
31pub trait AudioOutputPlugin: Plugin + Send + Sync {
32    async fn start(&self, rx: mpsc::Receiver<CognitiveOutputAudio>) -> Result<(), String>;
33}
34
35#[async_trait]
36pub trait STTPlugin: Plugin + Send + Sync {
37    async fn start(
38        &self,
39        audio_rx: mpsc::Receiver<InputVoiceAudio>,
40        transcript_tx: mpsc::Sender<SpeechTranscript>,
41        speech_detected: SpeechDetected,
42    ) -> Result<(), String>;
43}
44
45#[async_trait]
46pub trait TTSPlugin: Plugin + Send + Sync {
47    async fn start(
48        &self,
49        speech_rx: broadcast::Receiver<CognitiveOutputSpeech>,
50        audio_tx: mpsc::Sender<CognitiveOutputAudio>,
51    ) -> Result<(), String>;
52}
53
54#[async_trait]
55pub trait DiarizationPlugin: Plugin + Send + Sync {
56    async fn start(
57        &self,
58        audio_rx: broadcast::Receiver<InputVoiceAudio>,
59        segment_tx: mpsc::Sender<SpeakerSegment>,
60    ) -> Result<(), String>;
61
62    fn heuristic(&self) -> Option<crate::speech_to_text::types::SpeakerHeuristicCallback> {
63        None
64    }
65}
66
67#[async_trait]
68pub trait ChatPlugin: Plugin + Send + Sync {
69    fn channel_context_schema() -> schemars::Schema
70    where
71        Self: Sized,
72    {
73        schemars::schema_for!(())
74    }
75
76    async fn start(
77        &self,
78        peer_input_text_tx: mpsc::Sender<crate::peer_input_text::types::PeerInputText>,
79        cognitive_output_text_rx: mpsc::Receiver<
80            crate::cognitive_output_text::types::CognitiveOutputText,
81        >,
82        cognitive_state_rx: broadcast::Receiver<crate::types::CognitiveStateUpdate>,
83        add_document_tx: Option<mpsc::Sender<crate::types::AddDocumentRequest>>,
84    ) -> Result<(), String>;
85}
86
87#[async_trait]
88pub trait GuiPlugin: Plugin + Send + Sync {
89    async fn start(
90        &self,
91        registries: std::sync::Arc<crate::types::ContextRegistries>,
92        error_rx: std::sync::mpsc::Receiver<String>,
93    ) -> Result<(), String>;
94}
95
96#[async_trait]
97pub trait DocumentsPlugin: Plugin + Send + Sync {
98    async fn start(
99        &self,
100        add_document_rx: mpsc::Receiver<crate::types::AddDocumentRequest>,
101    ) -> Result<(), String>;
102}
103
104#[async_trait]
105pub trait RetrospectiveConsolidationPlugin: Plugin + Send + Sync {
106    async fn start(
107        &self,
108        not_clear_memory_rx: watch::Receiver<crate::types::NotClearInteractionMemory>,
109        resolve_not_clear_tx: mpsc::Sender<crate::types::Timestamp>,
110    ) -> Result<(), String>;
111}
112
113pub trait PluginRegistry {
114    fn register_gui<P: GuiPlugin>(&mut self, plugin: std::sync::Arc<P>);
115    fn register_audio_input<P: AudioInputPlugin>(&mut self, plugin: std::sync::Arc<P>);
116    fn register_audio_output<P: AudioOutputPlugin>(&mut self, plugin: std::sync::Arc<P>);
117    fn register_stt<P: STTPlugin>(&mut self, plugin: std::sync::Arc<P>);
118    fn register_tts<P: TTSPlugin>(&mut self, plugin: std::sync::Arc<P>);
119    fn register_diarization<P: DiarizationPlugin>(&mut self, plugin: std::sync::Arc<P>);
120    fn register_chat<P: ChatPlugin>(&mut self, plugin: std::sync::Arc<P>);
121    fn register_documents<P: DocumentsPlugin>(&mut self, plugin: std::sync::Arc<P>);
122    fn register_interaction_observer<P: InteractionObserver>(&mut self, plugin: std::sync::Arc<P>);
123    fn register_rollout_controller<P: RolloutController>(&mut self, plugin: std::sync::Arc<P>);
124    fn register_retrospective_consolidation<P: RetrospectiveConsolidationPlugin>(
125        &mut self,
126        plugin: std::sync::Arc<P>,
127    );
128    fn register_camera<P: CameraPlugin>(&mut self, plugin: std::sync::Arc<P>);
129    fn register_context_provider<P: crate::types::ContextProvider>(
130        &mut self,
131        provider: std::sync::Arc<P>,
132    );
133    fn register_command<C: crate::types::Command>(&mut self, command: C);
134    fn register_tool<T: crate::types::Tool>(&mut self, tool: T);
135    fn register_call<P: CallPlugin>(
136        &mut self,
137        plugin: std::sync::Arc<P>,
138        capability: Option<&'static str>,
139    );
140    fn register_recorder<P: AudioRecorderPlugin>(&mut self, plugin: std::sync::Arc<P>);
141}
142
143use serde::{Deserialize, Serialize};
144
145#[derive(Debug, Clone, Serialize, Deserialize, Default)]
146pub struct EmptyPluginConfig {}
147
148#[async_trait]
149pub trait Plugin: Send + Sync + 'static {
150    /// Compile-time semantic description of this plugin's capability for the LLM.
151    const CAPABILITY: Option<&'static str> = None;
152
153    /// This is the method for instantiating plugins, allowing them to await
154    /// their database connections (via `context.store::<S>().await`) before returning.
155    ///
156    /// **Note on Configuration:** When calling `context.config()?` to extract your configuration struct,
157    /// ensure any optional fields in your struct are marked with `#[serde(default)]`. Otherwise,
158    /// omitted fields in the config file will cause strict deserialization errors.
159    async fn create(context: crate::types::PluginContext) -> Result<Self, String>
160    where
161        Self: Sized;
162
163    fn register<R: PluginRegistry + ?Sized>(self: std::sync::Arc<Self>, registry: &mut R)
164    where
165        Self: Sized;
166}
167
168#[async_trait]
169pub trait CallPlugin: Plugin + Send + Sync {
170    async fn start(
171        &self,
172        peer_input_text_rx: sync::broadcast::Receiver<crate::peer_input_text::types::PeerInputText>,
173        cognitive_output_text_tx: sync::mpsc::Sender<
174            crate::cognitive_output_text::types::CognitiveOutputText,
175        >,
176        last_voice_time_rx: sync::watch::Receiver<std::time::Instant>,
177        ai_speaking_rx: sync::watch::Receiver<bool>,
178        call_active_tx: sync::watch::Sender<bool>,
179    ) -> Result<(), String>;
180}
181
182#[async_trait]
183pub trait AudioRecorderPlugin: Plugin + Send + Sync {
184    async fn start(
185        &self,
186        call_active_rx: watch::Receiver<bool>,
187        input_voice_audio_rx: broadcast::Receiver<InputVoiceAudio>,
188    ) -> Result<(), String>;
189}
190
191#[async_trait]
192pub trait CameraPlugin: Plugin + Send + Sync {
193    async fn start(
194        &self,
195        video_tx: crate::sync::watch::Sender<crate::types::CameraInputFrame>,
196    ) -> Result<(), String>;
197}
198
199#[async_trait]
200pub trait RolloutController: Plugin + Send + Sync {
201    async fn start(&self, rollout_tx: watch::Sender<crate::types::Timestamp>)
202    -> Result<(), String>;
203}
204
205#[async_trait]
206pub trait InteractionObserver: Plugin + Send + Sync {
207    async fn start(
208        &self,
209        interaction_rx: mpsc::Receiver<crate::types::ObservedInteraction>,
210    ) -> Result<(), String>;
211}
synapto_interface/lib.rs

synapto_interface/
lib.rs
