1#![forbid(unsafe_code)]
2#![cfg(unix)]
3#![cfg_attr(test, allow(clippy::unwrap_used))]
4#![cfg_attr(
5 test,
6 allow(
7 clippy::disallowed_methods,
8 clippy::disallowed_types,
9 reason = "tests spawn, join and drain freely; production sites carry their own expectation"
10 )
11)]
12mod asset;
13mod cache_key;
14mod collection;
15mod config;
16mod conversation;
17#[cfg(feature = "fauxlack")]
18pub mod fauxlack;
19mod follow;
20mod gateway;
21mod jobs;
22mod memory;
23use memory::journal;
24mod meter_restore;
25mod metering;
26mod probe;
27mod progress;
28mod proxy;
29mod pull_request_links;
30mod routes;
31mod session;
32mod threads;
33mod transport;
34mod wake;
35
36pub mod cli;
37
38use std::{
39 collections::{BTreeMap, HashMap},
40 future::Future,
41 path::Path,
42 sync::Arc,
43 time::Duration,
44};
45
46use dekopon_broker_protocol::BrokerClient;
47use dekopon_config::LocalCatalog;
48use dekopon_process::ShellThreads;
49use thiserror::Error;
50use tokio::{sync::mpsc, task::JoinSet};
51
52pub use config::{
53 CONFIG_API_VERSION, ConfigApiVersion, ConfigError, ConfigProblem, ConversationMatchConfig,
54 DEFAULT_FORGET_AFTER, DEFAULT_STOP_WORDS, GatewaydConfig, HARD_MAX_CONFIG_BYTES,
55 IndicatorSurface, JournalConfig, KeepAliveConfig, LivenessConfig, LivenessMode,
56 LivenessOverride, LivenessSettings, MemoryConfig, MemoryPolicy, MemoryScope, MemoryWindow,
57 ProgressSurface, ReactionConfig, RecallSource, ResolvedConfig, ResolvedJournal, ResolvedRoute,
58 ResolvedTelemetry, SeenReaction, SlackExperience, TelemetryConfig, TemplateOverrides,
59 TransportConfig, WorkingReaction,
60};
61pub use probe::{NotProbed, ProbeError, ProbeProblem, ProbeScope};
62pub use routes::{RouteError, RouteProblem};
63pub use session::SessionError;
64pub use transport::TransportError;
65pub use wake::WakeStoreError;
66
67use crate::{
68 asset::AssetStore,
69 config::render_problems,
70 conversation::ConversationStore,
71 routes::RoutingTable,
72 session::{
73 ConfiguredModels, ModelCache, ModelCredentialError, SessionGate, SessionRunner,
74 model_bearer_token,
75 },
76 transport::{
77 AssetFetcher, CancelRequest, ChatDriver, ChatTransport, ThreadOwnership, TransportEvent,
78 discord::{DiscordTransport, MessageContent},
79 github::GithubTransport,
80 local::LocalTransport,
81 slack::SlackTransport,
82 telegram::TelegramTransport,
83 whatsapp::WhatsappTransport,
84 },
85};
86
87pub const OTEL_TRACE_FILTER: &str = "dekopon_gatewayd=trace,dekopon_agent=trace,dekopon_process=trace,dekopon_shell=trace,dekopon_model=trace,gateway=debug,prompt=debug,model=debug,asset=debug,shell=debug,job=debug,broker=debug,provider=debug,http=debug,credential=debug,memory=debug,telemetry=debug,hyper=off,h2=off,reqwest=off,tungstenite=off,tokio_tungstenite=off";
88
89const INBOUND_BUFFER: usize = 64;
90const ONE_SHOT_ASSET_IDLE: Duration = Duration::from_secs(60 * 60);
91
92#[must_use]
93pub fn current_uid() -> u32 {
94 rustix::process::geteuid().as_raw()
95}
96
97pub async fn telemetry_settings(
98 config_path: impl AsRef<Path>,
99 uid: u32,
100) -> Result<Option<ResolvedTelemetry>, GatewaydError> {
101 Ok(config::load(config_path, uid).await?.telemetry)
102}
103
104#[cfg(test)]
105async fn boot_candidate(config_path: &Path) -> Result<(Candidate, Prepared), GatewaydError> {
106 boot_candidate_with_service(config_path, &config::InProcess::default()).await
107}
108
109async fn boot_candidate_with_service(
110 config_path: &Path,
111 service: &config::InProcess,
112) -> Result<(Candidate, Prepared), GatewaydError> {
113 let (candidate, report) = load_candidate_with_service(
114 config_path,
115 None,
116 config::LoadMode::Boot,
117 None,
118 None,
119 service,
120 )
121 .await;
122 let Some((candidate, prepared)) = candidate.and_then(|mut candidate| {
123 candidate
124 .prepared
125 .take()
126 .map(|prepared| (candidate, prepared))
127 }) else {
128 return Err(GatewaydError::from_problems(report.problems));
129 };
130 if !candidate
131 .sources
132 .still_current(config_path, &candidate.config.catalog_path, current_uid())
133 .await?
134 {
135 return Err(config::ConfigError::ChangedDuringLoad.into());
136 }
137 Ok((candidate, prepared))
138}
139
140pub async fn run<F>(
141 config_path: impl AsRef<Path>,
142 shutdown: F,
143 shell_threads: u32,
144) -> Result<(), GatewaydError>
145where
146 F: Future<Output = ()> + Send,
147{
148 run_with_service(
149 config_path.as_ref(),
150 shutdown,
151 &config::InProcess::default(),
152 shell_threads,
153 )
154 .await
155}
156
157#[cfg(feature = "fauxlack")]
158pub async fn run_fauxlack<F>(
159 config_path: impl AsRef<Path>,
160 shutdown: F,
161 fauxlack: fauxlack::Fauxlack,
162 shell_threads: u32,
163) -> Result<(), GatewaydError>
164where
165 F: Future<Output = ()> + Send,
166{
167 run_with_service(
168 config_path.as_ref(),
169 shutdown,
170 &config::InProcess {
171 fauxlack: Some(fauxlack),
172 },
173 shell_threads,
174 )
175 .await
176}
177
178#[expect(
179 clippy::too_many_lines,
180 reason = "gateway boot owns resident transports and reload owners until shutdown"
181)]
182async fn run_with_service<F>(
183 config_path: &Path,
184 shutdown: F,
185 service: &config::InProcess,
186 shell_threads: u32,
187) -> Result<(), GatewaydError>
188where
189 F: Future<Output = ()> + Send,
190{
191 let (
192 candidate,
193 Prepared {
194 transports: built_transports,
195 generation,
196 },
197 ) = boot_candidate_with_service(config_path, service).await?;
198 let config = &candidate.config;
199
200 let broker_client = BrokerClient::new(
201 &config.broker.socket_path,
202 config.broker.server_uid,
203 config.broker.frame,
204 )
205 .map_err(GatewaydError::BrokerProbe)?;
206 let capabilities = broker_client
207 .capabilities()
208 .await
209 .map_err(GatewaydError::BrokerProbe)?;
210 let (broker_digest, outcome) = broker_client
211 .observe_generation()
212 .await
213 .map_err(GatewaydError::BrokerProbe)?;
214 if outcome != dekopon_broker_protocol::GenerationOutcome::Active {
215 return Err(GatewaydError::BrokerGeneration);
216 }
217 tracing::info!(
218 event = "gateway_broker_ready",
219 capability.count = capabilities.len(),
220 "gateway.broker.ready"
221 );
222 let (journal, boot) = match open_journal(config.journal.clone(), &config.memory).await? {
223 Some((journal, boot)) => (Some(journal), Some(boot)),
224 None => (None, None),
225 };
226 let follow =
227 load_follow_table(&journal, config.routes.iter().any(|route| route.follow)).await?;
228 generation.set_follow(follow.clone());
229 let mut drivers: BTreeMap<String, Arc<dyn ChatDriver>> = BTreeMap::new();
230 let mut asset_fetchers: HashMap<String, Arc<dyn AssetFetcher>> = HashMap::new();
231 let mut thread_ownership: HashMap<String, Arc<dyn ThreadOwnership>> = HashMap::new();
232 let (sender, receiver) = mpsc::channel::<TransportEvent>(INBOUND_BUFFER);
233 let mut readers = JoinSet::new();
234 for transport in built_transports {
235 let transport = register_transport(
236 transport,
237 &mut drivers,
238 &mut asset_fetchers,
239 &mut thread_ownership,
240 );
241 readers.spawn(read_transport(Box::new(transport), sender.clone()));
242 }
243
244 let journal_dir = config.journal.as_ref().map(|journal| journal.dir.clone());
245 let cache_keys = tokio::task::spawn_blocking(move || match journal_dir {
246 Some(dir) => cache_key::CacheKeySecret::load_or_create(&dir),
247 None => cache_key::CacheKeySecret::generate(),
248 })
249 .await
250 .map_err(GatewaydError::TransportTask)??;
251 if journal.is_some() {
252 for route in generation.load().routes.routes() {
253 memory::generation::report_policy(route);
254 }
255 }
256 let wakes = match config.wakes.clone() {
257 Some(wakes) => Some(Arc::new(
258 tokio::task::spawn_blocking(move || wake::WakeStore::open(&wakes))
259 .await
260 .map_err(GatewaydError::TransportTask)??,
261 )),
262 None => None,
263 };
264 let configured = Arc::new(ConfiguredModels::default());
265 let metering = Arc::new(metering::build(
266 &config.metering,
267 dekopon_model_token_governor::Metering::system_clock(),
268 ));
269 let assets_path = config.assets.clone();
270 let conversations = config.sessions.max_conversations;
271 let budget = config.sessions.asset_retention_bytes;
272 let journal_assets = journal.clone();
273 let assets = tokio::task::spawn_blocking(move || {
274 let store = AssetStore::with_retention(conversations, ONE_SHOT_ASSET_IDLE, budget);
275 let store = match assets_path {
276 Some(path) => store
277 .with_storage(&path)
278 .map_err(|error| GatewaydError::AssetStore { error })?,
279 None => store,
280 };
281 if let Some(journal) = journal_assets {
282 store
283 .attach_journal(&journal)
284 .map_err(|error| GatewaydError::Journal {
285 kind: match error {
286 journal::JournalError::Io { kind } => kind,
287 journal::JournalError::Corrupt => std::io::ErrorKind::InvalidData,
288 },
289 })?;
290 }
291 store
292 .report_startup()
293 .map_err(|error| GatewaydError::AssetStore { error })?;
294 Ok::<_, GatewaydError>(Arc::new(store))
295 })
296 .await
297 .map_err(GatewaydError::TransportTask)??;
298 let models = Arc::new(ModelCache::new(
299 Arc::clone(&configured) as Arc<dyn session::ModelFactory>
300 ));
301 let engine = journal.as_ref().map(|journal| {
302 memory::engine::channel(
303 Arc::clone(journal),
304 Arc::clone(&assets),
305 Arc::default(),
306 config.memory.summaries.queue,
307 config.memory.summaries.concurrency,
308 )
309 });
310 let thread_owner = threads::Owner::new(shell_threads);
311 let runner = Arc::new(SessionRunner {
312 pending_notices: Default::default(),
313 sealed_conversations: Default::default(),
314 broker: config.broker.clone(),
315 models: Arc::clone(&models),
316 summaries: engine.as_ref().map(|(sender, _)| sender.clone()),
317 gate: SessionGate::new(config.sessions.max_concurrent),
318 reply_on_busy: config.sessions.reply_on_busy,
319 conversations: ConversationStore::with_cache_keys(
320 config.sessions.max_conversations,
321 cache_keys,
322 ),
323 follow,
324 journal,
325 assets: Arc::clone(&assets),
326 asset_fetchers,
327 thread_ownership,
328 wakes,
329 jobs: Arc::new(jobs::Jobs::new(
330 config.sessions.max_jobs,
331 Arc::clone(&thread_owner),
332 )),
333 metering: Arc::clone(&metering),
334 shell_threads: ShellThreads::new(shell_threads, tokio::runtime::Handle::current()),
335 thread_owner: Arc::clone(&thread_owner),
336 });
337 let (mut owned_tasks, boot_proxy) =
338 start_metering_tasks(config, &candidate.routes, &configured, &metering)?;
339
340 tracing::info!(
341 event = "gateway_started",
342 transport.count = config.transports.len(),
343 route.count = candidate.routes.len(),
344 "gateway.started"
345 );
346
347 let mut terminal = Ok(());
348 let stopped = async {
349 terminal = supervise_transports(&mut readers, shutdown).await;
350 readers.abort_all();
351 };
352 let shutdown_grace = config.shutdown_grace;
353 let collector = collection::Collector::new(&config.transports, config.sessions.max_concurrent);
354 let proxy_listener = config.proxy.clone();
355 let runner_threads = runner.shell_threads.clone();
356 let gateway = gateway::Gateway::serving(
357 runner,
358 generation.clone(),
359 Arc::new(BTreeMap::new()),
360 Arc::new(drivers),
361 )
362 .with_reload_owners(
363 config_path.to_path_buf(),
364 candidate,
365 broker_digest,
366 gateway::ResidentModels {
367 credentials: Arc::clone(&configured),
368 metering,
369 proxy: boot_proxy,
370 },
371 )?;
372 if let Some(resolved) = &proxy_listener {
373 proxy::listen(resolved, gateway.proxy_router(), &mut owned_tasks)
374 .await
375 .map_err(GatewaydError::Proxy)?;
376 }
377 if let Some((sender, dispatcher)) = engine {
378 dispatcher.start(
379 generation.clone(),
380 models,
381 Arc::clone(&thread_owner),
382 runner_threads,
383 &mut owned_tasks,
384 );
385 if let Some(boot) = boot {
386 enqueue_boot(&sender, boot).await;
387 }
388 }
389 let outcome = gateway
390 .serve(receiver, stopped, shutdown_grace, collector)
391 .await;
392 readers.abort_all();
393 owned_tasks.shutdown().await;
394 thread_owner.shutdown().await;
395 configured.drain_credentials().await;
396 assets.shutdown().await;
397 while let Some(result) = readers.join_next().await {
398 match result {
399 Ok(Err(problem)) => tracing::warn!(
400 event = "gateway_transport_stopped",
401 transport = %problem.transport,
402 category = problem.source.category(),
403 "gateway.transport.stopped"
404 ),
405 Err(error) if !error.is_cancelled() => tracing::error!(
406 event = "gateway_transport_task_failed", error = %error,
407 "gateway.transport.task.failed"
408 ),
409 Ok(Ok(())) | Err(_) => {}
410 }
411 }
412 drop(sender);
413 if let Err(error) = terminal {
414 tracing::error!(
415 event = "gateway_stopped",
416 reason = "transport-failed",
417 "gateway.stopped"
418 );
419 return Err(error);
420 }
421 match outcome {
422 ServeOutcome::Shutdown => {
423 tracing::info!(
424 event = "gateway_stopped",
425 reason = "shutdown",
426 "gateway.stopped"
427 );
428 Ok(())
429 }
430 ServeOutcome::TransportsLost => {
434 tracing::error!(
435 event = "gateway_stopped",
436 reason = "transports-lost",
437 "gateway.stopped"
438 );
439 Err(GatewaydError::TransportsLost)
440 }
441 }
442}
443
444async fn enqueue_boot(sender: &memory::engine::Sender, boot: memory::journal::records::Boot) {
445 let enqueued = boot.tasks.len();
446 for task in boot.tasks {
447 sender.enqueue(task).await;
448 }
449 tracing::info!(
450 event = "gateway_memory_startup",
451 memory.journals.reset = boot.reset,
452 memory.slots.requeued = boot.requeued,
453 memory.slots.enqueued = enqueued,
454 memory.slots.overflowed = boot.overflowed,
455 memory.boot.crashed_files = boot.crashed_files,
456 "gateway.memory.startup"
457 );
458}
459
460async fn open_journal(
461 config: Option<config::ResolvedJournal>,
462 memory: &config::ResolvedSettings,
463) -> Result<Option<(Arc<journal::Journal>, memory::journal::records::Boot)>, GatewaydError> {
464 let Some(config) = config else {
465 return Ok(None);
466 };
467 let max_age = std::time::Duration::from_millis(memory.retention.max_age_ms.get());
468 let agent_ages = memory
469 .agents
470 .iter()
471 .map(|(id, agent)| (id.as_str().to_owned(), agent.max_age))
472 .collect();
473 let agent_bytes = memory
474 .agents
475 .iter()
476 .map(|(id, agent)| (id.as_str().to_owned(), agent.max_bytes.get()))
477 .collect();
478 let max_bytes_per_agent = memory.retention.max_bytes_per_agent.get();
479 let queue = memory.summaries.queue;
480 let journal = tokio::task::spawn_blocking(move || {
481 let (mut journal, boot) = journal::Journal::boot(
482 &config.dir,
483 max_age,
484 config.retention,
485 config.seal_retention,
486 queue,
487 std::time::SystemTime::now(),
488 )?;
489 journal.agent_ages = agent_ages;
490 journal.agent_bytes = agent_bytes;
491 journal.max_bytes_per_agent = max_bytes_per_agent;
492 journal.enforce_caps()?;
493 Ok::<_, journal::JournalError>((journal, boot))
494 })
495 .await
496 .map_err(GatewaydError::TransportTask)?
497 .map_err(|error| GatewaydError::Journal {
498 kind: match error {
499 journal::JournalError::Io { kind } => kind,
500 journal::JournalError::Corrupt => std::io::ErrorKind::InvalidData,
501 },
502 })?;
503 Ok(Some((Arc::new(journal.0), journal.1)))
504}
505
506async fn load_follow_table(
507 journal: &Option<Arc<journal::Journal>>,
508 enabled: bool,
509) -> Result<Option<Arc<follow::FollowTable>>, GatewaydError> {
510 if !enabled && journal.is_none() {
511 return Ok(None);
512 }
513 let journal = Arc::clone(journal.as_ref().ok_or(GatewaydError::Journal {
514 kind: std::io::ErrorKind::InvalidInput,
515 })?);
516 let table = tokio::task::spawn_blocking(move || follow::FollowTable::load(journal))
517 .await
518 .map_err(GatewaydError::TransportTask)?
519 .map_err(|error| GatewaydError::Journal {
520 kind: match error {
521 journal::JournalError::Io { kind } => kind,
522 journal::JournalError::Corrupt => std::io::ErrorKind::InvalidData,
523 },
524 })?;
525 Ok(Some(Arc::new(table)))
526}
527
528fn start_metering_tasks(
529 config: &ResolvedConfig,
530 routes: &RoutingTable,
531 configured: &ConfiguredModels,
532 metering: &Arc<dekopon_model_token_governor::Metering>,
533) -> Result<(JoinSet<()>, Option<Arc<dekopon_model_proxy::ModelProxy>>), GatewaydError> {
534 let mut owned_tasks = JoinSet::new();
535 for subject in proxy::listed_guests(config.proxy.as_ref()) {
536 tracing::warn!(
537 event = "gateway_proxy_guest_models_deprecated",
538 vm.subject = subject,
539 "gateway.proxy.guest_models_deprecated"
540 );
541 }
542 let boot_proxy = config
543 .proxy
544 .as_ref()
545 .map(|_| proxy::build(routes.guests(), &config.models, configured, metering))
546 .transpose()
547 .map_err(GatewaydError::Proxy)?;
548 meter_restore::spawn(
549 metering,
550 config.metering.restore.clone(),
551 config
552 .telemetry
553 .as_ref()
554 .and_then(|telemetry| telemetry.settings.ca_certificate().map(<[u8]>::to_vec)),
555 &mut owned_tasks,
556 );
557 Ok((owned_tasks, boot_proxy))
558}
559
560fn guest_model_warnings(config: &ResolvedConfig) -> impl Iterator<Item = CheckWarning> + '_ {
561 proxy::listed_guests(config.proxy.as_ref()).map(|subject| CheckWarning::GuestModels {
562 subject: subject.to_owned(),
563 })
564}
565
566#[derive(Debug, Default)]
567pub struct CheckReport {
568 pub problems: Vec<GatewaydError>,
569 pub warnings: Vec<CheckWarning>,
570 pub(crate) raw: config::RawCandidate,
571}
572
573#[derive(Debug, Error)]
574pub enum CheckWarning {
575 #[error("baseline routes did not bind, so every model is probed")]
576 BaselineRoutes {
577 #[source]
578 source: Box<GatewaydError>,
579 },
580 #[error("transport {transport} reads {variable} from the environment at boot")]
581 TransportCredential { transport: String, variable: String },
582 #[error("model {model} reads {variable} from the environment at boot")]
583 ModelCredential { model: String, variable: String },
584 #[error("persistent memory has no journal: window-only, memory tools unavailable")]
585 MemoryWithoutJournal,
586 #[error(
587 "proxy guest {subject:?} lists models, which is deprecated: drop the key to run it on its agent's model"
588 )]
589 GuestModels { subject: String },
590 #[error("{stage} not checked: a stage it needs failed")]
591 Skipped { stage: Stage },
592 #[error("model {model} not probed: {reason}")]
593 NotProbed { model: String, reason: NotProbed },
594 #[error("{} did not load, so every model is probed", .path.display())]
595 Baseline {
596 path: std::path::PathBuf,
597 #[source]
598 source: ConfigError,
599 },
600}
601
602#[derive(Clone, Copy, Debug, Eq, PartialEq)]
603pub enum Stage {
604 Catalog,
605 Routes,
606 Credentials,
607 Probe,
608}
609
610impl std::fmt::Display for Stage {
611 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
612 formatter.write_str(match self {
613 Self::Catalog => "agent catalog",
614 Self::Routes => "routes",
615 Self::Credentials => "credentials",
616 Self::Probe => "vendor probe",
617 })
618 }
619}
620
621pub async fn check(
622 config_path: impl AsRef<Path>,
623 catalog_path: Option<&Path>,
624 probe: Option<&ProbeScope>,
625) -> CheckReport {
626 load_candidate(
627 config_path.as_ref(),
628 catalog_path,
629 config::LoadMode::Check,
630 probe.map(|scope| (scope, &probe::Vendor::LIVE)),
631 )
632 .await
633 .1
634}
635
636pub(crate) struct Candidate {
637 pub(crate) sources: config::GatewaySources,
638 pub(crate) config: ResolvedConfig,
639 pub(crate) routes: RoutingTable,
640 pub(crate) prepared: Option<Prepared>,
641}
642
643pub(crate) async fn load_candidate(
644 config_path: &Path,
645 catalog_path: Option<&Path>,
646 mode: config::LoadMode,
647 probe: Option<(&ProbeScope, &probe::Vendor)>,
648) -> (Option<Candidate>, CheckReport) {
649 load_candidate_against(config_path, catalog_path, mode, probe, None).await
650}
651
652pub(crate) async fn load_candidate_against(
653 config_path: &Path,
654 catalog_path: Option<&Path>,
655 mode: config::LoadMode,
656 probe: Option<(&ProbeScope, &probe::Vendor)>,
657 active: Option<(&ResolvedConfig, &config::GatewaySources, &RoutingTable)>,
658) -> (Option<Candidate>, CheckReport) {
659 let service = active
660 .map(|(config, _, _)| config.service.clone())
661 .unwrap_or_default();
662 load_candidate_with_service(config_path, catalog_path, mode, probe, active, &service).await
663}
664
665async fn load_candidate_with_service(
666 config_path: &Path,
667 catalog_path: Option<&Path>,
668 mode: config::LoadMode,
669 probe: Option<(&ProbeScope, &probe::Vendor)>,
670 active: Option<(&ResolvedConfig, &config::GatewaySources, &RoutingTable)>,
671 service: &config::InProcess,
672) -> (Option<Candidate>, CheckReport) {
673 let mut report = CheckReport::default();
674 let before_config = config::source_snapshot(config_path, None, current_uid()).await;
675 let (mut config, refusals) =
676 config::load_reporting_with_service(config_path, current_uid(), mode, service).await;
677 report
678 .problems
679 .extend(refusals.into_iter().map(GatewaydError::from));
680 if probe.is_some()
681 && mode == config::LoadMode::Check
682 && let Some(candidate) = &mut config
683 && let Err(error) = candidate.load_model_ca_bundles()
684 {
685 report.problems.push(error.into());
686 config = None;
687 }
688 let catalog_path = catalog_path.or(config.as_ref().map(|config| config.catalog_path.as_path()));
689 let catalog = match catalog_path {
690 Some(path) => match LocalCatalog::load(path) {
691 Ok(catalog) => Some(catalog),
692 Err(error) => {
693 report.problems.push(GatewaydError::Catalog(error));
694 None
695 }
696 },
697 None => {
698 report.skipped(Stage::Catalog);
699 None
700 }
701 };
702 let raw_catalog = match (&catalog, catalog_path) {
703 (Some(catalog), _) => {
704 Some(config::source_snapshot(catalog.source(), Some(catalog), current_uid()).await)
705 }
706 (None, Some(path)) => Some(config::source_snapshot(path, None, current_uid()).await),
707 (None, None) => None,
708 };
709 report.raw = config::RawCandidate::of(
710 before_config.as_ref(),
711 raw_catalog.as_ref().map(Result::as_ref),
712 );
713 let routes = match (&config, &catalog) {
714 (Some(config), Some(catalog)) => match RoutingTable::bind(config, catalog) {
715 Ok(routes) => Some(routes),
716 Err(error) => {
717 report.problems.push(error.into());
718 None
719 }
720 },
721 (None, _) | (_, None) => {
722 report.skipped(Stage::Routes);
723 None
724 }
725 };
726 let prepared = match (&config, &routes) {
727 (Some(config), Some(routes)) => {
728 if mode == config::LoadMode::Check
729 && config.journal.is_none()
730 && routes.has_persistent_memory()
731 {
732 report.warnings.push(CheckWarning::MemoryWithoutJournal);
733 }
734 if mode == config::LoadMode::Check {
735 report.warnings.extend(guest_model_warnings(config));
736 }
737 if active.is_none() {
738 credentials(config, routes, mode, &mut report)
739 } else {
740 let problems = model_credential_problems(routes.bound_models(), model_bearer_token);
741 if !problems.is_empty() {
742 report.problems.push(GatewaydError::Startup { problems });
743 }
744 None
745 }
746 }
747 (None, _) | (_, None) => {
748 report.skipped(Stage::Credentials);
749 None
750 }
751 };
752 let reload_scope = ProbeScope::Every;
753 let probe = probe.or_else(|| active.map(|_| (&reload_scope, &probe::Vendor::LIVE)));
754 match (probe, &config) {
755 (Some((scope, vendor)), Some(config)) => {
756 probe::probe(
757 config,
758 routes.as_ref(),
759 scope,
760 active.map(|(config, _, routes)| (config, routes)),
761 vendor,
762 &mut report,
763 )
764 .await;
765 }
766 (Some(_), None) => report.skipped(Stage::Probe),
767 (None, _) => {}
768 }
769 let candidate = match (config, routes, catalog) {
770 (Some(config), Some(routes), Some(catalog)) if report.problems.is_empty() => {
771 let after_config = config::source_snapshot(config_path, None, current_uid()).await;
772 let after_catalog =
773 config::source_snapshot(catalog.source(), Some(&catalog), current_uid()).await;
774 match (before_config, after_config, after_catalog) {
775 (Ok(before), Ok(after), Ok(catalog_after)) if before == after => {
776 let sources = config::GatewaySources {
777 ca_bundles: config.ca_sources.clone(),
778 config: after,
779 catalog: catalog_after,
780 };
781 let changes = active.map(|(_, old, _)| old.changes(&sources)).transpose();
782 match changes {
783 Ok(Some(changes)) if !changes.restart_required.is_empty() => {
784 report.problems.push(
785 config::ConfigError::RestartRequired {
786 categories: changes.restart_required,
787 }
788 .into(),
789 );
790 None
791 }
792 Ok(_) => Some(Candidate {
793 sources,
794 config,
795 routes,
796 prepared,
797 }),
798 Err(error) => {
799 report.problems.push(error.into());
800 None
801 }
802 }
803 }
804 (Err(error), _, _) | (_, Err(error), _) | (_, _, Err(error)) => {
805 report.problems.push(error.into());
806 None
807 }
808 _ => {
809 report
810 .problems
811 .push(config::ConfigError::ChangedDuringLoad.into());
812 None
813 }
814 }
815 }
816 _ => None,
817 };
818 (candidate, report)
819}
820
821impl CheckReport {
822 fn skipped(&mut self, stage: Stage) {
823 self.warnings.push(CheckWarning::Skipped { stage });
824 }
825}
826
827fn credentials(
828 config: &ResolvedConfig,
829 routes: &RoutingTable,
830 mode: config::LoadMode,
831 report: &mut CheckReport,
832) -> Option<Prepared> {
833 match mode {
834 config::LoadMode::Boot => match prepare(config, routes) {
835 Ok(prepared) => return Some(prepared),
836 Err(error) => report.problems.push(error),
837 },
838 config::LoadMode::Check => {
839 for spec in &config.transports {
840 report
841 .warnings
842 .extend(
843 transport_credential_variables(spec)
844 .into_iter()
845 .map(|variable| CheckWarning::TransportCredential {
846 transport: spec.name().to_owned(),
847 variable: variable.to_owned(),
848 }),
849 );
850 }
851 for model in routes.bound_models() {
852 let mut required = None;
853 let missing = session::model_bearer_token_with(model, |variable| {
854 required = Some(variable.to_owned());
855 None
856 })
857 .is_err();
858 if let (true, Some(variable)) = (missing, required) {
859 report.warnings.push(CheckWarning::ModelCredential {
860 model: model.name().to_owned(),
861 variable,
862 });
863 }
864 }
865 }
866 }
867 None
868}
869
870fn transport_credential_variables(spec: &TransportConfig) -> Vec<&str> {
871 match spec {
872 TransportConfig::SlackSocketMode {
873 app_token_env,
874 bot_token_env,
875 ..
876 } => vec![app_token_env, bot_token_env],
877 TransportConfig::DiscordGateway { bot_token_env, .. }
878 | TransportConfig::TelegramLongPoll { bot_token_env, .. } => vec![bot_token_env],
879 TransportConfig::WhatsappCloudApi {
880 app_secret_env,
881 verify_token_env,
882 access_token_env,
883 ..
884 } => vec![app_secret_env, verify_token_env, access_token_env],
885 TransportConfig::GithubWebhook { secret_env, .. } => vec![secret_env],
886 TransportConfig::Fauxlack { .. } | TransportConfig::Local { .. } => Vec::new(),
887 }
888}
889
890#[derive(Clone, Copy, Debug, Eq, PartialEq)]
891enum ServeOutcome {
892 Shutdown,
893 TransportsLost,
894}
895
896#[cfg(test)]
897pub(crate) use tests::serve;
898
899fn observe_session(result: Result<(), tokio::task::JoinError>) {
900 if let Err(error) = result
901 && !error.is_cancelled()
902 {
903 tracing::error!(
904 event = "gateway_session_task_failed",
905 "gateway.session.task.failed"
906 );
907 }
908}
909
910fn spawn_tick(
911 runner: &Arc<SessionRunner>,
912 routes: &RoutingTable,
913 store: &Arc<wake::WakeStore>,
914 ticks: &mut JoinSet<Option<wake::Fired>>,
915 tick: wake::Tick,
916) -> Result<(), threads::Failure> {
917 let store = Arc::clone(store);
918 let Some(admission) = runner
920 .gate
921 .admit_probe(("wake".to_owned(), tick.id().to_string()))
922 else {
923 tracing::info!(
924 event = "gateway_wake_tick_skipped",
925 wake.id = %tick.id(),
926 reason = "busy",
927 "gateway.wake.tick.skipped"
928 );
929 return Ok(());
930 };
931 let ticket = runner
932 .thread_owner
933 .reserve(&runner.shell_threads)
934 .map_err(|limit| {
935 tracing::info!(
936 event = "gateway_wake_tick_skipped",
937 wake.id = %tick.id(),
938 reason = "shell-threads",
939 limit.maximum = runner.thread_owner.maximum,
940 error = ?limit,
941 "gateway.wake.tick.skipped"
942 );
943 threads::Failure::Limit(limit)
944 })?;
945 let shell = routes.route_for_anchor(tick.anchor()).map(|route| {
946 dekopon_shell::Interpreter::new(
947 dekopon_process::Limits {
948 max_capability_calls: route.limits.max_capability_calls,
949 timeout: route.script_timeout,
950 ..dekopon_process::Limits::default()
951 },
952 runner.shell_threads.clone(),
953 )
954 });
955 let job_limits = routes.route_for_anchor(tick.anchor()).and_then(|route| {
956 route.job_timeout.map(|timeout| dekopon_process::Limits {
957 max_capability_calls: route.limits.max_capability_calls,
958 timeout,
959 ..dekopon_process::Limits::default()
960 })
961 });
962 let jobs = Arc::clone(&runner.jobs);
963 let broker = runner.broker.clone();
964 let runtime = tokio::runtime::Handle::current();
965 let result = ticket.spawn(threads::Work::Tick, move || {
966 let _admission = admission;
967 wake::run_tick(
968 tick,
969 &store,
970 &broker,
971 &runtime,
972 shell,
973 job_limits.map(|limits| (jobs, limits)),
974 )
975 }).map_err(|error| {
976 tracing::error!(event = "gateway_wake_task_failed", error.kind = ?error.kind(), "gateway.wake.task.failed");
977 threads::Failure::Spawn { kind: error.kind() }
978 })?;
979 ticks.spawn(async move {
980 match result.await {
981 Ok(Ok(fired)) => fired,
982 Ok(Err(failure)) => {
983 tracing::error!(event = "gateway_wake_task_failed", failure = failure.category(), error = %failure, "gateway.wake.task.failed");
984 None
985 }
986 Err(_closed) => {
987 tracing::error!(event = "gateway_wake_task_failed", failure = "cancelled", "gateway.wake.task.failed");
988 None
989 }
990 }
991 });
992 Ok(())
993}
994
995fn cancel_session(runner: &Arc<SessionRunner>, request: &CancelRequest) {
996 runner.jobs.cancel_owner(request);
997 match runner.gate.cancel(request).running.ignored_reason() {
998 None => tracing::info!(
999 event = "gateway_session_stop_requested",
1000 transport = %request.transport,
1001 via = via_label(request.via),
1002 "gateway.session.stop.requested"
1003 ),
1004 Some(reason) => tracing::debug!(
1007 event = "gateway_session_stop_ignored",
1008 transport = %request.transport,
1009 reason,
1010 "gateway.session.stop.ignored"
1011 ),
1012 }
1013}
1014
1015const fn via_label(via: dekopon_agent::CancelVia) -> &'static str {
1016 match via {
1017 dekopon_agent::CancelVia::NativeStop => "native-stop",
1018 dekopon_agent::CancelVia::Button => "button",
1019 dekopon_agent::CancelVia::StopReply => "stop-reply",
1020 }
1021}
1022
1023fn register_transport(
1024 transport: Box<dyn ChatTransport>,
1025 drivers: &mut BTreeMap<String, Arc<dyn ChatDriver>>,
1026 asset_fetchers: &mut HashMap<String, Arc<dyn AssetFetcher>>,
1027 thread_ownership: &mut HashMap<String, Arc<dyn ThreadOwnership>>,
1028) -> transport::recovery::RecoveringTransport {
1029 let transport = transport::recovery::RecoveringTransport::new(transport);
1030 let name = transport.name().to_owned();
1031 drivers.insert(name.clone(), transport.driver());
1032 if let Some((sibling, driver)) = transport.sibling_driver() {
1033 drivers.insert(sibling, driver);
1034 }
1035 if let Some((sibling, fetcher)) = transport.sibling_fetcher() {
1036 asset_fetchers.insert(sibling, fetcher);
1037 }
1038 if let Some(fetcher) = transport.asset_fetcher() {
1039 asset_fetchers.insert(name.clone(), fetcher);
1040 }
1041 if let Some(ownership) = transport.thread_ownership() {
1042 thread_ownership.insert(name, ownership);
1043 }
1044 transport
1045}
1046
1047async fn read_transport(
1048 mut transport: Box<dyn ChatTransport>,
1049 sender: mpsc::Sender<TransportEvent>,
1050) -> Result<(), TransportConnectProblem> {
1051 let result = async {
1052 let identity = transport.connect().await?;
1053 if sender
1054 .send(TransportEvent::Connected {
1055 name: transport.name().to_owned(),
1056 identity,
1057 })
1058 .await
1059 .is_err()
1060 {
1061 return Ok(());
1062 }
1063 loop {
1064 let event = transport.next().await?;
1065 if sender.send(event).await.is_err() {
1066 return Ok(());
1067 }
1068 }
1069 }
1070 .await;
1071 result.map_err(|source| TransportConnectProblem {
1072 transport: transport.name().to_owned(),
1073 source,
1074 })
1075}
1076
1077async fn supervise_transports<F>(
1078 readers: &mut JoinSet<Result<(), TransportConnectProblem>>,
1079 shutdown: F,
1080) -> Result<(), GatewaydError>
1081where
1082 F: Future<Output = ()> + Send,
1083{
1084 tokio::select! {
1085 biased;
1086 () = shutdown => Ok(()),
1087 result = readers.join_next() => match result {
1088 Some(Ok(Err(problem))) => Err(GatewaydError::TransportConnect { problems: vec![problem] }),
1089 Some(Err(source)) => Err(GatewaydError::TransportTask(source)),
1090 Some(Ok(Ok(()))) | None => Err(GatewaydError::TransportsLost),
1091 }
1092 }
1093}
1094
1095fn model_credential_problems<'a>(
1096 models: impl IntoIterator<Item = &'a config::ModelConfig>,
1097 mut resolve: impl FnMut(
1098 &config::ModelConfig,
1099 ) -> Result<Option<String>, session::ModelCredentialError>,
1100) -> Vec<StartupProblem> {
1101 models
1102 .into_iter()
1103 .filter_map(|model| resolve(model).err().map(StartupProblem::ModelCredential))
1104 .collect()
1105}
1106
1107#[cfg(test)]
1108mod openrouter_startup_tests {
1109 use super::*;
1110
1111 #[test]
1112 fn missing_and_blank_openrouter_credentials_are_startup_problems_not_config_problems() {
1113 let model = config::ModelConfig::Openrouter {
1114 ca_bundles: Default::default(),
1115 name: "router".into(),
1116 model: "vendor/model".into(),
1117 api_key_env: "OPENROUTER_API_KEY".into(),
1118 timeout_ms: 1000,
1119 classes: Vec::new(),
1120 modalities: Vec::new(),
1121 generation: None,
1122 reasoning: None,
1123 routing: None,
1124 cache: None,
1125 };
1126 for value in [None, Some(std::ffi::OsString::from(" "))] {
1127 let problems = model_credential_problems([&model], |model| {
1128 session::model_bearer_token_with(model, |variable| {
1129 assert_eq!(variable, "OPENROUTER_API_KEY");
1130 value.clone()
1131 })
1132 });
1133 assert!(
1134 matches!(problems.as_slice(), [StartupProblem::ModelCredential(error)] if error.variable == "OPENROUTER_API_KEY" && error.model == "router")
1135 );
1136 }
1137 let problems = model_credential_problems([&model], |model| {
1138 session::model_bearer_token_with(model, |_| Some("synthetic-key".into()))
1139 });
1140 assert!(problems.is_empty());
1141 }
1142}
1143
1144fn prepare(config: &ResolvedConfig, routes: &RoutingTable) -> Result<Prepared, GatewaydError> {
1145 let mut problems = model_credential_problems(routes.bound_models(), model_bearer_token);
1146 let generation = gateway::CurrentGeneration::boot(
1147 Arc::new(routes.clone()),
1148 Arc::new(config.stop_words.clone()),
1149 );
1150 let whatsapp_binds = config
1151 .transports
1152 .iter()
1153 .filter_map(|spec| match spec {
1154 TransportConfig::WhatsappCloudApi { bind, .. } => Some(*bind),
1155 TransportConfig::Fauxlack { .. }
1156 | TransportConfig::GithubWebhook { .. }
1157 | TransportConfig::SlackSocketMode { .. }
1158 | TransportConfig::DiscordGateway { .. }
1159 | TransportConfig::TelegramLongPoll { .. }
1160 | TransportConfig::Local { .. } => None,
1161 })
1162 .collect::<std::collections::HashSet<_>>();
1163 let mut github_mounts = HashMap::new();
1164 for spec in &config.transports {
1165 if let TransportConfig::GithubWebhook {
1166 name,
1167 bind,
1168 secret_env,
1169 } = spec
1170 && whatsapp_binds.contains(bind)
1171 {
1172 match transport::read_credential(secret_env) {
1173 Ok(secret) => {
1174 github_mounts.insert(*bind, (name.clone(), secret));
1175 }
1176 Err(source) => problems.push(StartupProblem::Transport {
1177 transport: name.clone(),
1178 source,
1179 }),
1180 }
1181 }
1182 }
1183 let mut transports = Vec::with_capacity(config.transports.len());
1184 for spec in &config.transports {
1185 if matches!(spec, TransportConfig::GithubWebhook { bind, .. } if whatsapp_binds.contains(bind))
1186 {
1187 continue;
1188 }
1189 match build_transport(spec, config, &generation, &mut github_mounts) {
1190 Ok(transport) => transports.push(transport),
1191 Err(source) => problems.push(StartupProblem::Transport {
1192 transport: spec.name().to_owned(),
1193 source,
1194 }),
1195 }
1196 }
1197 if problems.is_empty() {
1198 Ok(Prepared {
1199 transports,
1200 generation,
1201 })
1202 } else {
1203 Err(GatewaydError::Startup { problems })
1204 }
1205}
1206
1207pub(crate) struct Prepared {
1208 transports: Vec<Box<dyn ChatTransport>>,
1209 generation: gateway::CurrentGeneration,
1210}
1211
1212fn build_transport(
1213 spec: &TransportConfig,
1214 config: &ResolvedConfig,
1215 generation: &gateway::CurrentGeneration,
1216 github_mounts: &mut HashMap<std::net::SocketAddr, (String, String)>,
1217) -> Result<Box<dyn ChatTransport>, TransportError> {
1218 let surface = |liveness: &LivenessConfig| {
1219 let mut settings = liveness.settings();
1220 if config
1221 .routes
1222 .iter()
1223 .filter(|route| route.transport == spec.name())
1224 .flat_map(|route| route.indicators.values())
1225 .any(|resolved| resolved.settings.mode == LivenessMode::Native)
1226 {
1227 settings.mode = LivenessMode::Native;
1228 }
1229 settings
1230 };
1231 Ok(match spec {
1232 TransportConfig::Fauxlack {
1233 name, server, bot, ..
1234 } => {
1235 #[cfg(feature = "fauxlack")]
1236 {
1237 let service = config
1238 .service
1239 .fauxlack
1240 .as_ref()
1241 .ok_or(TransportError::Response)?;
1242 Box::new(transport::fauxlack::FauxlackTransport::new(
1243 name.clone(),
1244 service.server(server),
1245 bot.clone(),
1246 ))
1247 }
1248 #[cfg(not(feature = "fauxlack"))]
1249 {
1250 let _ = (name, server, bot);
1251 return Err(TransportError::Response);
1252 }
1253 }
1254 TransportConfig::GithubWebhook {
1255 name,
1256 secret_env,
1257 bind,
1258 } => Box::new(GithubTransport::new(
1259 name.clone(),
1260 *bind,
1261 transport::read_credential(secret_env)?,
1262 generation.clone(),
1263 None,
1264 )),
1265 TransportConfig::SlackSocketMode {
1266 name,
1267 app_token_env,
1268 bot_token_env,
1269 experience,
1270 endpoint,
1271 ..
1272 } => Box::new(
1273 SlackTransport::new(
1274 name.clone(),
1275 endpoint
1276 .clone()
1277 .unwrap_or_else(|| config::SLACK_ENDPOINT.to_owned()),
1278 transport::read_credential(app_token_env)?,
1279 transport::read_credential(bot_token_env)?,
1280 *experience,
1281 )?
1282 .with_generation(generation.clone()),
1283 ),
1284 TransportConfig::DiscordGateway {
1285 name,
1286 bot_token_env,
1287 message_content,
1288 liveness,
1289 endpoint,
1290 } => Box::new(DiscordTransport::new(
1291 name.clone(),
1292 endpoint
1293 .clone()
1294 .unwrap_or_else(|| config::DISCORD_ENDPOINT.to_owned()),
1295 transport::read_credential(bot_token_env)?,
1296 if *message_content {
1297 MessageContent::Requested
1298 } else {
1299 MessageContent::Withheld
1300 },
1301 surface(liveness),
1302 )?),
1303 TransportConfig::WhatsappCloudApi {
1304 name,
1305 app_secret_env,
1306 verify_token_env,
1307 access_token_env,
1308 bind,
1309 callback_path,
1310 waba_id,
1311 phone_number_id,
1312 graph_api_version,
1313 liveness,
1314 graph_endpoint,
1315 ..
1316 } => {
1317 let whatsapp = WhatsappTransport::new(
1318 name.clone(),
1319 *bind,
1320 callback_path.clone(),
1321 waba_id.clone(),
1322 phone_number_id.clone(),
1323 graph_api_version.clone(),
1324 graph_endpoint
1325 .clone()
1326 .unwrap_or_else(|| config::WHATSAPP_GRAPH_ENDPOINT.to_owned()),
1327 transport::read_credential(app_secret_env)?,
1328 transport::read_credential(verify_token_env)?,
1329 transport::read_credential(access_token_env)?,
1330 surface(liveness),
1331 )?;
1332 match github_mounts.remove(bind) {
1333 Some((name, secret)) => Box::new(GithubTransport::new(
1334 name,
1335 *bind,
1336 secret,
1337 generation.clone(),
1338 Some(whatsapp),
1339 )),
1340 None => Box::new(whatsapp),
1341 }
1342 }
1343 TransportConfig::TelegramLongPoll {
1344 name,
1345 bot_token_env,
1346 liveness,
1347 endpoint,
1348 ..
1349 } => Box::new(TelegramTransport::new(
1350 name.clone(),
1351 endpoint
1352 .clone()
1353 .unwrap_or_else(|| config::TELEGRAM_ENDPOINT.to_owned()),
1354 transport::read_credential(bot_token_env)?,
1355 surface(liveness),
1356 )?),
1357 TransportConfig::Local {
1358 name,
1359 socket_path,
1360 liveness,
1361 } => Box::new(LocalTransport::new(
1362 name.clone(),
1363 socket_path.clone(),
1364 surface(liveness),
1365 )),
1366 })
1367}
1368
1369#[derive(Debug, Error)]
1370pub enum GatewaydError {
1371 #[error("could not open the attachment store: {error}")]
1372 AssetStore {
1373 error: dekopon_model::asset::BlobError,
1374 },
1375 #[error("gateway configuration is invalid")]
1376 Config(#[from] ConfigError),
1377 #[error("gateway agent catalog is unavailable or invalid")]
1378 Catalog(#[source] dekopon_config::ConfigError),
1379 #[error("gateway route cannot be satisfied")]
1380 Route(#[from] RouteError),
1381 #[error("{}", render_problems(.problems))]
1382 Startup { problems: Vec<StartupProblem> },
1383 #[error("broker is not reachable; start dekopon-brokerd before the gateway")]
1384 BrokerProbe(#[source] dekopon_broker_protocol::ClientError),
1385 #[error("broker generation was not active")]
1386 BrokerGeneration,
1387 #[error("{}", render_problems(.problems))]
1388 TransportConnect {
1389 problems: Vec<TransportConnectProblem>,
1390 },
1391 #[error("chat transport task failed")]
1392 TransportTask(#[source] tokio::task::JoinError),
1393 #[error("every chat transport ended; the gateway can no longer be reached")]
1394 TransportsLost,
1395 #[error("conversation journal directory is unusable: {kind}")]
1396 Journal { kind: std::io::ErrorKind },
1397 #[error(transparent)]
1398 CacheKeySecret(#[from] cache_key::CacheKeySecretError),
1399 #[error(transparent)]
1400 WakeStore(#[from] WakeStoreError),
1401 #[error("guest model proxy could not start")]
1402 Proxy(#[source] proxy::ProxyStartError),
1403 #[error(transparent)]
1404 Probe(Box<ProbeProblem>),
1405 #[error("{}", render_problems(.problems))]
1406 Problems { problems: Vec<GatewaydError> },
1407}
1408
1409impl GatewaydError {
1410 fn from_problems(mut problems: Vec<Self>) -> Self {
1411 match (problems.pop(), problems.is_empty()) {
1412 (Some(only), true) => only,
1413 (Some(last), false) => {
1414 problems.push(last);
1415 Self::Problems { problems }
1416 }
1417 (None, _) => Self::Problems { problems },
1418 }
1419 }
1420}
1421
1422#[derive(Debug, Error)]
1423#[error("chat transport {transport} failed")]
1424pub struct TransportConnectProblem {
1425 pub transport: String,
1426 #[source]
1427 pub source: TransportError,
1428}
1429
1430#[derive(Debug, Error)]
1431pub enum StartupProblem {
1432 #[error("configured model credential is unavailable")]
1433 ModelCredential(#[source] ModelCredentialError),
1434 #[error("chat transport {transport} could not be prepared")]
1435 Transport {
1436 transport: String,
1437 #[source]
1438 source: TransportError,
1439 },
1440}
1441
1442#[cfg(test)]
1443mod tests;